diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala new file mode 100644 index 000000000..a56d5cb5c --- /dev/null +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.bridge + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import com.sksamuel.elastic4s.requests.searches.SearchRequest +import com.sksamuel.elastic4s.requests.searches.aggs.{AbstractAggregation, Aggregation} +import com.sksamuel.elastic4s.requests.searches.aggs.pipeline.BucketScriptPipelineAgg + +import scala.jdk.CollectionConverters._ + +/** The PARAMS of a `bucket_script`'s script, which the elastic4s body builder drops. + * + * πŸ”΄ elastic4s writes a `bucket_script` as `"script": ""` -- the bare source string + * (`BucketScriptPipelineAggBuilder`, the same in elastic4s 6.7, 7.17, 8.18 and 9.0) -- while a + * `bucket_selector` gets the whole script object. So a per-group calculation reading the request + * clock, `CURRENT_DATE - MAX(d)` or `NOW() - MAX(ts)`, was sent WITHOUT the `__now__` its script + * reads, and Elasticsearch failed it; the same clock in a HAVING worked. MEASURED on 8.18.3. + * + * The script is sent as the object Elasticsearch documents -- `source`, `lang`, `params` -- for + * exactly the `bucket_script`s that carry params, so the source stays the SAME text from one + * request to the next and Elasticsearch compiles it once: inlining the clock instead would compile + * a new script on every request, against the cluster's compilation rate limit. Every other body is + * returned untouched, unparsed. + */ +object BucketScriptParams { + + private val mapper = new ObjectMapper() + + /** The params of every `bucket_script` of `aggs` that carries some, by aggregation name, at any + * depth. + */ + private def withParams(aggs: Seq[AbstractAggregation]): Map[String, Map[String, Any]] = + aggs.flatMap { + case script: BucketScriptPipelineAgg if script.script.params.nonEmpty => + Seq(script.name -> script.script.params) + case parent: Aggregation => withParams(parent.subaggs).toSeq + case _ => Nil + }.toMap + + /** `body`, the JSON of `search`, with the params of its `bucket_script`s bound. */ + def bind(body: String, search: SearchRequest): String = { + val params = withParams(search.aggs) + if (params.isEmpty) body + else { + val root = mapper.readTree(body) + bind(root, params) + mapper.writeValueAsString(root) + } + } + + private def bind(node: JsonNode, params: Map[String, Map[String, Any]]): Unit = + node match { + case obj: ObjectNode => + obj.fields().asScala.toList.foreach { entry => + (params.get(entry.getKey), entry.getValue) match { + case (Some(values), agg: ObjectNode) if agg.has("bucket_script") => + agg.get("bucket_script") match { + case bucketScript: ObjectNode if bucketScript.path("script").isTextual => + val script = mapper.createObjectNode() + script.put("source", bucketScript.get("script").asText()) + script.put("lang", "painless") + val bound = script.putObject("params") + values.foreach { case (name, value) => + bound.set[JsonNode](name, mapper.valueToTree[JsonNode](value)) + } + bucketScript.set[JsonNode]("script", script) + case _ => + } + case _ => + } + bind(entry.getValue, params) + } + case array: ArrayNode => array.elements().asScala.foreach(bind(_, params)) + case _ => + } +} diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index d441615cc..0dc21aad3 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -162,7 +162,8 @@ object ElasticAggregation { allAggregations: Map[String, SQLAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -218,14 +219,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -475,7 +476,8 @@ object ElasticAggregation { allElasticAggregations: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = @@ -492,7 +494,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { @@ -720,7 +722,8 @@ object ElasticAggregation { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Option[Aggregation] = request.having.flatMap(_.criteria) match { case Some(criteria) if request.wholeTableHaving && aggs.nonEmpty => diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 37f64eefd..667992081 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -48,7 +48,8 @@ case class ElasticBridge(filter: ElasticFilter) { currentQuery: Option[ElasticBoolQuery] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { filter match { case boolQuery: ElasticBoolQuery => diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index 89e86e969..b196a3923 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.requests.searches.queries.Query @@ -24,7 +24,8 @@ case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index 3337d604e..4bad08ea7 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -49,5 +49,10 @@ case class ElasticSearchRequest( def hasTransformExtendedStats: Boolean = SearchBodySerializer.hasTransformExtendedStats(search) def query: String = - serializer.serialize(search).replace("\"version\":true,", "") /*FIXME*/ + // the params of a per-group calculation's script, which the body builder drops: the request + // clock `CURRENT_DATE - MAX(d)` reads (`BucketScriptParams`) + BucketScriptParams.bind( + serializer.serialize(search).replace("\"version\":true,", ""), /*FIXME*/ + search + ) } diff --git a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index c680ad446..d3a5a2851 100644 --- a/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -77,7 +77,8 @@ package object bridge { innerHitsName: String )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { @@ -156,7 +157,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -184,7 +186,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) @@ -259,7 +262,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations @@ -472,10 +476,16 @@ package object bridge { // SearchBodySerializer in implicit scope of this conversion; with none in scope the default // argument applies (the one-argument elastic4s builder, refusing a transform-bearing // extended_stats). Same defaulted-implicit pattern as `contextType`. + // + // `target` is the same seam for SCRIPTS: the Elasticsearch major the client module talks to + // (`PainlessTarget`), which every conversion below hands to the `PainlessContext`s it creates. + // With none in scope a script renders as for the majors from 7 on; the ES 6 client modules + // give 6, whose query scripts compare a `date` doc value differently. implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -537,7 +547,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): SearchRequest = { import request._ @@ -600,7 +611,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -621,7 +632,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -687,7 +698,8 @@ package object bridge { request: MultiSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): MultiSearchRequest = { // Story 22.6 β€” a NAMED backstop, never a silent `UNION ALL`. Only `UNION ALL` is an // Elasticsearch `_msearch`; every other set operator needs the relational engine and is @@ -741,9 +753,10 @@ package object bridge { private[bridge] def scriptQueryOf(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Query = { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = criteria.painless(Some(context)) scriptQuery( now(Script(script = s"$context$script").lang("painless").scriptType("source")) @@ -757,7 +770,8 @@ package object bridge { implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import expression._ if (isAggregation) @@ -982,7 +996,7 @@ package object bridge { case _ => scriptQueryOf(expression) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -993,7 +1007,7 @@ package object bridge { ) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -1037,7 +1051,8 @@ package object bridge { implicit def inToQuery[R, T <: Value[R]](in: InExpr[R, T])(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { if (requiresScript(in.identifier)) return scriptQueryOf(in) import in._ @@ -1062,7 +1077,8 @@ package object bridge { between: BetweenExpr )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import between._ // ⚠️ NOT the two-operand form: `BetweenExpr.maybeValue` is always a `FromTo`, never an @@ -1215,14 +1231,16 @@ package object bridge { implicit def criteriaToQuery(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { ElasticCriteria(criteria).asQuery() } implicit def criteriaToNode(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): JsonNode = { queryToJson(criteriaToQuery(criteria)) } @@ -1240,7 +1258,8 @@ package object bridge { )(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): Seq[ElasticAggregation] = { import query._ statement @@ -1280,7 +1299,10 @@ package object bridge { aggregation.copy( sources = l.sources, query = Some( - serializer.serialize(body).replace("\"version\":true,", "") /*FIXME*/ + BucketScriptParams.bind( + serializer.serialize(body).replace("\"version\":true,", ""), /*FIXME*/ + body + ) ) ) }) diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala new file mode 100644 index 000000000..f6927d6e9 --- /dev/null +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime +import scala.jdk.CollectionConverters._ + +/** A per-group calculation that reads the request clock (`CURRENT_DATE - MAX(d)`, `NOW() - + * MAX(ts)`) is sent WITH the clock: its `bucket_script` carries the script object -- `source`, + * `lang`, `params.__now__` -- where the elastic4s builder writes the bare source string and drops + * the params (`BucketScriptParams`). Every other `bucket_script` is sent as it was. + * + * The clock is a PARAM, not text spliced into the source: the source is the same from one request + * to the next, so Elasticsearch compiles it once. + */ +class BucketScriptParamsSpec extends AnyFlatSpec with Matchers { + + import scala.language.implicitConversions + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def timestamp: Long = now + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) + case other => throw new IllegalArgumentException(s"Not a single search: $other") + } + + private val mapper = new ObjectMapper() + + private def bucketScripts(sql: String): Seq[JsonNode] = { + val request: ElasticSearchRequest = SelectStatement(sql) + def all(node: JsonNode): Seq[JsonNode] = + Option(node.get("bucket_script")).toSeq ++ node.elements().asScala.toSeq.flatMap(all) + all(mapper.readTree(request.query)) + } + + "a per-group calculation reading the clock" should "send the clock as a script param" in { + Seq( + "SELECT g, CURRENT_DATE - MAX(d) AS x FROM t GROUP BY g", + "SELECT g, NOW() - MAX(ts) AS x FROM t GROUP BY g", + "SELECT g, DATEDIFF(CURRENT_DATE, MAX(d)) AS x FROM t GROUP BY g" + ).foreach { sql => + withClue(s"[$sql] ") { + val scripts = bucketScripts(sql) + scripts should have size 1 + val script = scripts.head.get("script") + script.isObject shouldBe true + script.get("source").asText() should include("params.__now__") + script.get("lang").asText() shouldBe "painless" + script.get("params").get("__now__").asLong() shouldBe now + } + } + } + + "a per-group calculation without the clock" should "be sent as it always was" in { + val scripts = bucketScripts("SELECT g, MAX(d) - MIN(d) AS x FROM t GROUP BY g") + scripts should have size 1 + scripts.head.get("script").isTextual shouldBe true + } +} diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala new file mode 100644 index 000000000..e385013b6 --- /dev/null +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime + +/** The Elasticsearch major the client module puts in implicit scope (`PainlessTarget`) reaches + * EVERY query script the bridge's conversions create: a WHERE, a script field, an aggregation over + * an expression. + * + * The probe is a comparison between a `date` doc value and another date, which Elasticsearch 6.8 + * alone needs normalised to UTC: with 6 in scope the body carries the normalisation, with any + * other major -- or none -- it is the very body no target produces. + */ +class PainlessTargetSpec extends AnyFlatSpec with Matchers { + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + private val normalisation = ".withZoneSameInstant(ZoneId.of('Z'))" + + private val schema = Parser( + "CREATE TABLE t (id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, n INT, PRIMARY KEY (id))" + ) match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(ss: SingleSearch) => + val r = ss.update(Some(schema)) + r.validateResolved() shouldBe Right(()) + r + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + /** The body `requestToElasticSearchRequest` sends. (`sqlQueryToAggregations`, the other door, + * parses its statement from text: no schema, so no type to normalise a comparison by.) + */ + private def body(sql: String, target: PainlessTarget): String = + requestToElasticSearchRequest(resolved(sql))( + now, + PainlessContextType.Query, + SearchBodySerializer.Default, + target + ).query + + private val venues: Seq[String] = Seq( + // a WHERE: a script query, alone and inside a bool + "SELECT id FROM t WHERE CAST(ts AS DATE) = d", + "SELECT id FROM t WHERE CURRENT_DATE > d AND n > 1", + // a script field + "SELECT id, CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END AS c FROM t", + // an aggregation over an expression + "SELECT SUM(CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END) AS s FROM t", + // the WHERE of an aggregate query + "SELECT g, COUNT(*) AS c FROM t WHERE CAST(ts AS DATE) = d GROUP BY g" + ) + + "a script comparing a date doc value" should + "carry the UTC normalisation when the client module gives Elasticsearch 6" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + body(sql, PainlessTarget(6)) should include(normalisation) + } + } + } + + it should "be the body no target produces for Elasticsearch 7, 8 and 9" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + val default = body(sql, PainlessTarget.Default) + default should not include normalisation + Seq(7, 8, 9).foreach(major => body(sql, PainlessTarget(major)) shouldBe default) + body(sql, PainlessTarget(6)).replace(normalisation, "") shouldBe default + } + } + } +} diff --git a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index b0f1f2888..446a2421e 100644 --- a/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2525,7 +2525,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate().plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", | "params": { | "__now__": 1767139200000 | } diff --git a/build.sbt b/build.sbt index 546ed3193..3e18e8141 100644 --- a/build.sbt +++ b/build.sbt @@ -167,6 +167,19 @@ lazy val log4jClosureSettings: Seq[Setting[_]] = Seq( } ) +/** The Elasticsearch major a CLIENT module is built against, generated from its own + * `elasticSearchVersion` so that no client source writes it by hand: the module's search-body + * serializer and the `PainlessTarget` its scripts are rendered for both read it. + */ +def clientBuildInfoSettings(pkg: String, obj: String): Seq[Setting[_]] = Seq( + buildInfoKeys := Seq[BuildInfoKey]( + BuildInfoKey("elasticVersion" -> elasticSearchVersion.value), + BuildInfoKey("elasticMajor" -> elasticSearchMajorVersion(elasticSearchVersion.value)) + ), + buildInfoPackage := pkg, + buildInfoObject := obj +) + lazy val licensing = project .in(file("licensing")) .configs(IntegrationTest) @@ -454,11 +467,13 @@ lazy val es6testkit = testkitProject(Versions.es6) lazy val es6rest = project .in(file("es6/rest")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.rest", "RestClientBuildInfo"), elasticSearchVersion := Versions.es6 ) .dependsOn( @@ -479,11 +494,13 @@ lazy val es6cli = lazy val es6jest = project .in(file("es6/jest")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.jest", "JestClientBuildInfo"), elasticSearchVersion := Versions.es6 ) .dependsOn( @@ -519,11 +536,13 @@ lazy val es7testkit = testkitProject(Versions.es7) lazy val es7rest = project .in(file("es7/rest")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.rest", "RestClientBuildInfo"), elasticSearchVersion := Versions.es7 ) .dependsOn( @@ -564,11 +583,13 @@ lazy val es8testkit = testkitProject(Versions.es8) lazy val es8java = project .in(file("es8/java")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.java", "JavaClientBuildInfo"), elasticSearchVersion := Versions.es8 ) .dependsOn( @@ -619,11 +640,13 @@ lazy val es9testkit = testkitProject( lazy val es9java = project .in(file("es9/java")) + .enablePlugins(BuildInfoPlugin) .configs(IntegrationTest) .settings( Defaults.itSettings, moduleSettings, log4jClosureSettings, + clientBuildInfoSettings("app.softnetwork.elastic.client.java", "JavaClientBuildInfo"), scalaVersion := scala213, crossScalaVersions := Seq(scala213), elasticSearchVersion := Versions.es9, diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala index 63cf62945..4d4cb1185 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientDelegator.scala @@ -327,6 +327,9 @@ trait ElasticClientDelegator extends ElasticClientApi with BulkTypes { override def loadSchema(index: String): ElasticResult[Schema] = delegate.loadSchema(index) + override private[client] def loadPatternSchema(pattern: String): ElasticResult[Schema] = + delegate.loadPatternSchema(pattern) + override def invalidateSchema(index: String): Unit = delegate.invalidateSchema(index) override def invalidateAllSchemas(): Unit = delegate.invalidateAllSchemas() diff --git a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala index cf027019d..1640f9e02 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/GatewayApi.scala @@ -1065,6 +1065,21 @@ class TableExecutor( case Some(from) => api.loadSchema(from) match { case ElasticSuccess(fromSchema) => + // πŸ”΄ A TYPE refusal (`MAX(d) / 1000`, `d + d2`, `n + 'x'`) is asked HERE, once + // the source's column types are known, before the target is created or -- `OR + // REPLACE` -- the existing one deleted. Its SELECT is refused anyway when it runs + // (`validateResolved`), but that runs after the destructive step, and since no + // type is refused when a statement is parsed (the lead's ruling of 2026-10-05), + // a type error would otherwise lose the user's index and put nothing back. + single.update(Some(fromSchema)).validateResolved().left.foreach { reason => + val error = ElasticError( + message = reason, + statusCode = Some(400), + operation = Some("schema") + ) + logger.error(s"❌ ${error.message}") + return Left(error) + } // we update the schema based on the DQL select clause fromSchema .mergeWithSearch(single) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index 8389fd858..c629bd280 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -34,6 +34,7 @@ import app.softnetwork.elastic.sql.query.{ Update } import app.softnetwork.elastic.sql.schema.{ + mergedSchema, GenericProcessor, IngestPipeline, Schema, @@ -250,6 +251,8 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { // #238 β€” a shard count cached for an expression this index is the stem of (a name probed // before it existed, `orders*` before the load) must not survive the creation invalidateShardCounts(Some(index)) + // ...nor a pattern's merged schema that did not read this index yet + invalidatePatternsOf(index) logger.info(s"βœ… Index '$index' created successfully") success case success @ ElasticSuccess(_) => @@ -305,6 +308,91 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { result } + /** The schema an index PATTERN or a comma list answers with (`FROM logs-*`, `FROM a,b`): the + * columns of every index it matches, merged field by field (`schema.mergedSchema`), so a + * statement over a pattern is typed as one over a single index is. A field the indices map + * differently is left untyped: nothing is refused on it, and Elasticsearch answers for it. + * + * ONE mapping read per pattern -- `GET ` answers every matched index at once -- through + * the SAME cache as [[loadSchema]], keyed by the pattern, so a hit costs no round trip. A + * pattern that matches no index answers a `404` failure: no schema is attached, and `SearchApi` + * remembers the miss for its TTL as it does an unknown index's. + * + * ⚠️ A merged schema carries the columns and nothing else -- no settings, no pipeline, no + * primary key: it types a QUERY, never a DDL statement. An elasticsql DDL statement that + * creates, alters or drops an index the pattern matches drops the entry, as it refreshes the + * single index's ([[invalidatePatternsOf]]): the next statement reads the members again. + */ + private[client] def loadPatternSchema(pattern: String): ElasticResult[Schema] = { + val now = System.currentTimeMillis() + var result: ElasticResult[Schema] = null + var fetched = false + schemaCache.compute( + pattern, + (_, existing) => + existing match { + case entry: CachedSchema if !entry.isExpired(now) => + logger.debug(s"πŸ“¦ Schema cache hit for '$pattern'") + result = ElasticSuccess(entry.schema) + entry + case _ => + fetched = true + fetchPatternSchema(pattern) match { + case success @ ElasticSuccess(schema) => + result = success + CachedSchema(schema, now, schemaCacheTtlMs) + case failure => + result = failure + existing + } + } + ) + if (fetched && schemaCache.size() > schemaCachePurgeThreshold) purgeExpiredSchemas(now) + result + } + + private def fetchPatternSchema(pattern: String): ElasticResult[Schema] = { + def notFound(message: String): ElasticResult[Schema] = + ElasticFailure( + ElasticError( + message = message, + statusCode = Some(404), + index = Some(pattern), + operation = Some("loadSchema") + ) + ) + pattern + .split(",") + .map(_.trim) + .find(part => validateIndexName(part, pattern = true).isDefined) match { + case Some(part) => return notFound(s"Invalid index pattern '$part' in '$pattern'") + case None => + } + executeGetIndex(pattern) match { + case ElasticSuccess(Some(json)) => + val root = mapper.readTree(json) + val members: Seq[(String, JsonNode)] = + if (root.has(pattern)) Seq(pattern -> root.get(pattern)) else Index.indexDocuments(root) + if (members.isEmpty) notFound(s"No index matches '$pattern'") + else { + val (schema, disagreements) = mergedSchema( + pattern, + members.map { case (name, document) => name -> Index(name, document).schema } + ) + if (disagreements.nonEmpty) + logger.info( + s"⚠️ The indices of '$pattern' disagree on ${disagreements.mkString(", ")}: " + + "statements are typed by Elasticsearch on those fields" + ) + ElasticSuccess(schema) + } + case ElasticSuccess(None) => notFound(s"No index matches '$pattern'") + case ElasticFailure(error) if error.statusCode.contains(404) => + notFound(s"No index matches '$pattern'") + case ElasticFailure(error) => ElasticFailure(error) + } + } + /** Drop every expired entry β€” each on ITS own clock, through the single `isExpired` rule β€” plus * the alias->target rows for exactly those entries. * @@ -362,10 +450,31 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { logger.debug(s"πŸ“¦ Schema cache invalidated for alias '$alias' (target '$index')") } + /** Drop every cached PATTERN schema ([[loadPatternSchema]]) whose pattern matches `index`: its + * merged columns read `index`'s mapping, so an elasticsql DDL statement that creates, alters or + * drops `index` changes the pattern's answer exactly as it changes the single index's -- and, + * like the single index's, it is read again on the next statement rather than served stale for + * its TTL. A pattern this one does not match keeps its entry. + */ + private def invalidatePatternsOf(index: String): Unit = + // (a pattern is never the subject of a DDL statement: its own entry, written through + // `updateSchema(, …)`, is not one of the entries an index's change invalidates) + if (!IndicesApi.isPattern(index)) + schemaCache + .keySet() + .asScala + .filter(key => IndicesApi.isPattern(key) && IndicesApi.patternMatches(key, index)) + .toList + .foreach { pattern => + schemaCache.remove(pattern) + logger.debug(s"πŸ“¦ Schema cache invalidated for pattern '$pattern' (index '$index')") + } + def updateSchema(index: String, schema: Schema): Unit = { val now = System.currentTimeMillis() schemaCache.put(index, CachedSchema(schema, now, resolveTtlMs(index, schema))) invalidateAliasesOf(index) + invalidatePatternsOf(index) // #238 β€” ALTER TABLE may have reindexed into a different shard count invalidateShardCounts(Some(index)) logger.debug(s"πŸ“¦ Schema cache updated for '$index'") @@ -375,6 +484,7 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { schemaCache.remove(index) val _ = schemaAliasTargets.remove(index) invalidateAliasesOf(index) + invalidatePatternsOf(index) invalidateShardCounts(Some(index)) // #238 β€” the sliced-paging shard counts follow the schema logger.info(s"πŸ—‘οΈ Schema cache invalidated for '$index'") } @@ -2158,3 +2268,28 @@ trait IndicesApi extends ElasticClientHelpers with SchemaCacheTtlApi { refresh: Boolean ): ElasticResult[Long] } + +object IndicesApi { + + /** A FROM expression that names several indices: a `*` wildcard or a comma list. */ + private[client] def isPattern(source: String): Boolean = + source.contains("*") || source.contains(",") + + /** Does the index `pattern` -- a comma list of names and `*` wildcards, as Elasticsearch reads + * one -- match `index`? An EXCLUSION (`-logs-old`) is not read: a pattern that matches `index` + * through another part is matched, and one matched only through a part an exclusion would remove + * is matched too -- reading a mapping again costs one request, serving it stale a wrong type. + */ + private[client] def patternMatches(pattern: String, index: String): Boolean = + pattern.split(",").iterator.map(_.trim).filter(p => p.nonEmpty && !p.startsWith("-")).exists { + part => + part + .split("\\*", -1) + .map(java.util.regex.Pattern.quote) + .mkString(".*") + .r + .pattern + .matcher(index) + .matches() + } +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 8b0cad2a4..c264deb36 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -95,15 +95,16 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC * without a temporal candidate β€” `baseType` is needed for every statement, and skipping would * have left the attach dead for nearly every query. Cost: one lookup per index per cache TTL. * - * Schema-absent path = verbatim (#276 AD-S4-1.2): the lookup is skipped when the FROM does not - * name exactly ONE concrete index (several sources, a `*` wildcard or a `,` list), when this - * client is not an [[IndicesApi]], and when the schema cannot be loaded (alias without a - * template, unknown index, cluster error or a thrown lookup) - * -- the literal is then forwarded exactly as before. The schema comes from - * [[IndicesApi.loadSchema]] 's 5-minute cache, so a miss costs one `GET ` per index per - * TTL. A literal that cannot be a date under a fully-understood mapping format is a `400` that - * names the literal and the field, instead of Elasticsearch's - * `search_phase_execution_exception`. + * Schema-absent path = verbatim (#276 AD-S4-1.2): the lookup is skipped when this client is not + * an [[IndicesApi]]; no schema is attached when it cannot be loaded (alias without a template, + * unknown index, cluster error or a thrown lookup) -- the literal is then forwarded exactly as + * before, and no TYPE is refused: Elasticsearch answers. A field the indices of a `*` pattern or + * a list of sources disagree on is attached untyped, with the same effect for that field alone. + * The schema comes from [[IndicesApi.loadSchema]] 's 5-minute cache (a pattern's, merged, from + * [[IndicesApi.loadPatternSchema]] 's, under the same key rules), so a miss costs one `GET + * ` per index -- or per pattern -- per TTL. A literal that cannot be a date under a + * fully-understood mapping format is a `400` that names the literal and the field, instead of + * Elasticsearch's `search_phase_execution_exception`. * * `loadSchema` caches successes only, so a source it cannot resolve -- an ALIAS on the es8/es9 * clients (`executeGetIndex` looks the alias up as a key and finds nothing), an unknown index @@ -177,11 +178,17 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC // That was correct while the only job was rewriting those literals, and is WRONG now that the // schema is also attached to the AST: almost no statement carries a temporal WHERE literal, // so the lookup would be skipped for almost every query and `baseType` would stay `Any`. + // An index PATTERN or a comma list -- `FROM logs-*`, `FROM a, b` (several sources searched as + // one) -- answers like the single index: the schema of the indices it matches, merged field by + // field (`IndicesApi.loadPatternSchema`), one mapping read per list, cached. Before, it attached + // NO schema, so `FROM zwild_*` typed every column `Any` and answered `MAX(d) - MIN(d)` in + // milliseconds where the single index answered days. phaseOne.sources.distinct match { - case Seq(source) if !source.contains("*") && !source.contains(",") => - this match { - case indices: IndicesApi if !schemaMissed(source) => - Try(indices.loadSchema(source)) match { + case sources if sources.nonEmpty => + val source = sources.mkString(",") + lookupSchema(sources) match { + case Some(lookup) => + lookup match { case Success(ElasticSuccess(schema)) => schemaMisses.remove(source) TemporalLiterals(phaseOne, schema) match { @@ -254,20 +261,41 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC ) ElasticResult.success(phaseOne) } - case _ => ElasticResult.success(phaseOne) + case None => ElasticResult.success(phaseOne) } case _ => ElasticResult.success(phaseOne) } } + /** THE schema lookup for a statement's sources, for every consumer -- the attach + * ([[resolveWithSchema]]) and the scope of a subquery ([[resolvedSchema]]) alike (#292): ONE + * concrete index reads its own mapping ([[IndicesApi.loadSchema]]); an index PATTERN, a comma + * list or several sources read the mapping of every index they match, merged field by field + * ([[IndicesApi.loadPatternSchema]]), under the key the miss is remembered by -- the sources + * joined with commas. + * + * `None` when no lookup is made: a client that is not an [[IndicesApi]], or a remembered miss. + */ + private def lookupSchema(sources: Seq[String]): Option[Try[ElasticResult[Schema]]] = { + val source = sources.mkString(",") + this match { + case indices: IndicesApi if !schemaMissed(source) => + val pattern = sources.size > 1 || source.contains("*") || source.contains(",") + Some(Try(if (pattern) indices.loadPatternSchema(source) else indices.loadSchema(source))) + case _ => None + } + } + /** The mapped type of a column PROJECTED BY AN INNER STATEMENT, when it can be known for free * (story 22.2). * - * `None` under every #306 skip condition β€” several sources, a wildcard, a client that is not an - * `IndicesApi`, a remembered miss, a failed or absent schema. The caller (`SubqueryResolver`) - * reads it for ONE decision: a `text` column cannot carry a terms aggregation, so mode P is - * declined for it. An unknown mapping therefore keeps the cheap default, and an Elasticsearch - * refusal propagates loudly with the inner statement's own message. + * `None` for a client that is not an `IndicesApi`, a remembered miss, a failed or absent schema; + * an index pattern or a list of sources answers from its merged schema, the one + * [[resolveWithSchema]] attaches (a field its indices disagree on is `Any`, which declines + * nothing). The caller (`SubqueryResolver`) reads it for ONE decision: a `text` column cannot + * carry a terms aggregation, so mode P is declined for it. An unknown mapping therefore keeps + * the cheap default, and an Elasticsearch refusal propagates loudly with the inner statement's + * own message. */ private[client] def innerColumnType(inner: SingleSearch, name: String): Option[SQLType] = resolvedSchema(inner).flatMap(_.find(name).map(_.dataType)) @@ -366,16 +394,16 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC outerMapping.name ) + /** The schema of a statement's sources, when it can be read: [[lookupSchema]], the one the attach + * reads -- an index pattern or a list of sources included (their merged schema), so a subquery + * over `logs-*` is scoped as one over a single index is. + */ private def resolvedSchema(s: SingleSearch): Option[Schema] = s.sources.distinct match { - case Seq(source) if !source.contains("*") && !source.contains(",") => - this match { - case indices: IndicesApi if !schemaMissed(source) => - Try(indices.loadSchema(source)) match { - case Success(ElasticSuccess(schema)) => Some(schema) - case _ => None - } - case _ => None + case sources if sources.nonEmpty => + lookupSchema(sources).flatMap { + case Success(ElasticSuccess(schema)) => Some(schema) + case _ => None } case _ => None } @@ -452,11 +480,18 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers with SchemaC case (failure, _) => failure } match { case ElasticSuccess(resolved) => - // Story 22.6 β€” with each branch's schema attached, bare columns are TYPED: re-run the - // pairwise compatibility check that passed vacuously at parse time (where a bare column is - // `Any`). This is the only type guard the family has β€” DuckDB casts implicitly across - // branches, so the relational engine is never a backstop either. - MultiSearch.branchTypes(resolved) match { + // Story 22.6 β€” with each branch's schema attached, bare columns are TYPED: this is where + // the pairwise compatibility check is asked. This is the only type guard the family has β€” + // DuckDB casts implicitly across branches, so the relational engine is never a backstop + // either. + // + // πŸ”΄ The lead's ruling of 2026-10-05: no TYPE refusal before the column types are known. + // Parsing no longer asks it at all, and with NO schema attached to any branch (a client + // that is not an `IndicesApi`, mappings that cannot be read or that disagree) nothing is + // refused on types: Elasticsearch answers, exactly as for a single statement. A branch + // without a schema keeps its bare columns `Any`, which pass. + (if (resolved.exists(_.schema.isDefined)) MultiSearch.branchTypes(resolved) + else Right(())) match { case Left(reason) => ElasticResult.failure( ElasticError( diff --git a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala index 420e2d76f..3f43458ba 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/metrics/MetricsElasticClient.scala @@ -317,6 +317,11 @@ class MetricsElasticClient( delegate.loadSchema(index) } + override private[client] def loadPatternSchema(pattern: String): ElasticResult[Schema] = + measureResult("loadSchema", Some(pattern.indices.mkString(","))) { + delegate.loadPatternSchema(pattern) + } + override def invalidateSchema(index: String): Unit = delegate.invalidateSchema(index) override def invalidateAllSchemas(): Unit = delegate.invalidateAllSchemas() diff --git a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala index b31b43ce3..03602ae19 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/RelationalClosureGuardSpec.scala @@ -22,6 +22,7 @@ import app.softnetwork.elastic.sql.query.{ ctesPresent, relationalClosureRequired, setOperationsPresent, + MultiSearch, SearchStatement, SingleSearch } @@ -364,15 +365,26 @@ class RelationalClosureGuardSpec extends AnyFlatSpec with Matchers { } } - /** Story 22.6 AD-4's second half: at parse time a bare column is `SQLTypes.Any` and passes, so - * the check must RE-RUN once schemas are attached. `NopeClientApi` attaches none, so this row - * pins the LITERAL case the parse-time half already rejects plus the seam's own wiring; the - * schema-attached case is covered by `TemporalLiteralSearchSpec`'s stub-schema idiom. + /** Story 22.6 AD-4: the branch TYPE check. The lead's ruling of 2026-10-05 -- no TYPE refusal + * before the column types are known -- took it off parsing entirely: the seam asks it once a + * branch's schema is attached, which `TemporalLiteralSearchSpec`'s stub-schema idiom pins, with + * nothing sent. `NopeClientApi` attaches no schema, so this row pins the rest: the statement + * PARSES, the rule the seam asks refuses it by name, and with no schema at all nothing is + * refused on types -- Elasticsearch answers. */ - it should "reject a typeable branch mismatch before any request is built" in { - Parser("SELECT 1 AS n FROM t UNION ALL SELECT 'a' AS n FROM u").swap.toOption - .map(_.msg) - .getOrElse("") should include("compatible types at column 1") + it should "ask the branch type check after resolution, and refuse nothing on types without a schema" in { + val sql = "SELECT 1 AS n FROM t UNION ALL SELECT 'a' AS n FROM u" + Parser(sql) match { + case Right(multi: MultiSearch) => + MultiSearch.branchTypes(multi.requests).swap.getOrElse("") should include( + "compatible types at column 1" + ) + case other => fail(s"[$sql] parsed as $other: no TYPE is refused at parse") + } + client().search(searchStatement(sql)) match { + case ElasticFailure(e) => fail(s"refused without a schema: ${e.message}") + case ElasticSuccess(_) => succeed + } } it should "refuse an INSERT ... SELECT carrying a set operation" in { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala index b04bf19ed..ea78b53e2 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SubqueryResolverSpec.scala @@ -633,6 +633,71 @@ class SubqueryResolverSpec extends AnyFlatSpec with Matchers { msg should include("must be QUALIFIED") } + /** A subquery over an index PATTERN is scoped by the schema the pattern is attached with -- the + * merged mapping of the indices it matches, through the ONE lookup the attach reads (#292) -- + * where it used to be read for a single concrete index only: over a pattern the bare-name rule + * passed vacuously, and a correlated reference ran as an uncorrelated one. A field the indices + * disagree on stays a column of the pattern (untyped), so it is never mistaken for an outer one. + */ + it should "scope a subquery over an index pattern by the pattern's merged mapping" in { + val client = seededClient() + client.updateSchema( + "members", + schema.Table( + "members", + columns = List( + schema.Column("id", SQLTypes.Keyword), + schema.Column("tier", SQLTypes.Keyword), + schema.Column("level", SQLTypes.Keyword) + ) + ) + ) + val (merged, disagreements) = schema.mergedSchema( + "orders_*", + Seq( + "orders_a" -> schema.Table( + "orders_a", + columns = List( + schema.Column("id", SQLTypes.Keyword), + schema.Column("cid", SQLTypes.Keyword), + schema.Column("tier", SQLTypes.Keyword) + ) + ), + "orders_b" -> schema.Table( + "orders_b", + columns = List( + schema.Column("id", SQLTypes.Keyword), + schema.Column("cid", SQLTypes.Keyword), + schema.Column("tier", SQLTypes.Int) + ) + ) + ) + ) + disagreements shouldBe Seq("'tier' (INT, KEYWORD)") + // the cache key a pattern's schema is read under is the pattern itself + client.updateSchema("orders_*", merged) + + // a name the pattern does not map while the outer index does: a correlated reference + val correlated = single( + "SELECT id FROM members m WHERE id IN (SELECT cid FROM orders_* o WHERE level = 'gold')" + ) + val correlatedInner = correlated.whereSubqueries.head.inner.getOrElse(fail("no body")) + client.innerColumnType(correlatedInner, "cid") shouldBe Some(SQLTypes.Keyword) + client + .scopeCorrelation(correlated, correlatedInner) + .getOrElse(fail("expected a rejection")) should include( + "'level' is not a column of 'orders_*'" + ) + + // the field the indices disagree on is still the pattern's own column, untyped + val own = single( + "SELECT id FROM members m WHERE id IN (SELECT cid FROM orders_* o WHERE tier = 'gold')" + ) + val ownInner = own.whereSubqueries.head.inner.getOrElse(fail("no body")) + client.innerColumnType(ownInner, "tier") shouldBe Some(SQLTypes.Any) + client.scopeCorrelation(own, ownInner) shouldBe None + } + /** πŸ”΄ Story 22.3b's DEFENSIVE arm. `GatewayApi.run(statement: Statement)` never calls * `validate()` and the seam's closure guard reads the STATEMENT, so a node handed straight to * this object must never have its correlated body executed as if it were self-contained. The diff --git a/documentation/sql/ddl_statements.md b/documentation/sql/ddl_statements.md index 3c032b3eb..5f6cdc415 100644 --- a/documentation/sql/ddl_statements.md +++ b/documentation/sql/ddl_statements.md @@ -545,9 +545,10 @@ value of the incoming document, not the temporal object a query sees, so `YEAR(c `DATE_TRUNC(...)`, `DATE_ADD(...)`, `DATE_FORMAT(...)` and `DATE_DIFF(...)` parse it first. Both shapes Elasticsearch accepts into a `date` field work β€” an ISO string (`"2025-01-10"`, `"2025-01-10 14:30:00"`, `"2025-01-10T14:30:00Z"`) and epoch milliseconds (`1736467200000`) β€” and -the column's DECLARED type chooses how a string is read, so a `DATE` column expects a date-only -spelling. `CURRENT_DATE`, `CURRENT_TIMESTAMP`, `NOW` and `TODAY` are the ingest time of the -document. +the column's DECLARED type chooses how a string is read: a `DATE` column reads a date-only +spelling as that day, and a date-time spelling (which Elasticsearch accepts into it) as the UTC day +of the instant it spells. `CURRENT_DATE`, `CURRENT_TIMESTAMP`, `NOW` and `TODAY` are the ingest time +of the document. > A document that does not carry the source field leaves the computed column absent, as before. diff --git a/documentation/sql/dql_statements.md b/documentation/sql/dql_statements.md index 1e5db4dc3..f300e78e2 100644 --- a/documentation/sql/dql_statements.md +++ b/documentation/sql/dql_statements.md @@ -645,7 +645,9 @@ The SQL Gateway executes `UNION ALL` using **Elasticsearch Multi‑Search (`_mse - `UNION ALL` does **not** sort or deduplicate results. - Column names in the final output are taken from the **first SELECT**. - All subsequent SELECTs must produce columns with the **same names**. -- Type mismatches should result in a validation error before execution. (⚠️ not implemented yet) +- A type mismatch between branches is refused before execution, once each branch's mapping is read + (`SELECT id FROM a UNION ALL SELECT amount FROM b`, a `keyword` against a number); with no + mapping to read, nothing is refused on types and Elasticsearch answers. --- @@ -1400,11 +1402,17 @@ GREATEST(e1, e2, ...) LEAST(e1, e2, ...) ``` -`GREATEST` returns the largest non-null numeric value among the given expressions; `LEAST` -returns the smallest. NULL arguments are ignored (ANSI semantics); the result is NULL only -when every argument is NULL. Both are emitted as Painless ternary chains over -`Math.max` / `Math.min`. They are row-level conditional functions, not aggregates β€” -`GREATEST(...) OVER (...)` is not supported. +`GREATEST` returns the largest non-null value among the given expressions; `LEAST` +returns the smallest. The arguments are all numeric, or all dates and timestamps (a `DATE` +compares as the start of its day, UTC; the result is a `DATE` over dates and a `TIMESTAMP` when +they mix). Over numbers the result is the common type of all the arguments on every row: the +widest whole type when every argument is a whole number, a `DOUBLE` as soon as one is fractional. +NULL arguments are ignored (ANSI semantics); the result is NULL only +when every argument is NULL. Both are emitted as Painless ternary chains β€” over whole numbers the +winner is chosen by a comparison, over fractional numbers by `Math.max` / `Math.min`, over dates +over the epoch milliseconds each one denotes. They are +conditional functions, not aggregates β€” `GREATEST(...) OVER (...)` is not supported β€” but over +aggregates they are computed per group (`GREATEST(MAX(a), MAX(b))` beside a `GROUP BY`). ```sql SELECT GREATEST(price_us, price_eu, price_uk) AS max_price FROM products; diff --git a/documentation/sql/functions_conditional.md b/documentation/sql/functions_conditional.md index c15bf783b..f2b01d13c 100644 --- a/documentation/sql/functions_conditional.md +++ b/documentation/sql/functions_conditional.md @@ -241,7 +241,8 @@ NULLIF(expr1, expr2) - `expr2` - second expression **Output:** -- Type of `expr1`, or NULL if equal +- Type of `expr1` β€” a quoted literal first is the value it is read as beside `expr2`, and has its + type: `NULLIF('2024-01-31', ts)` is a `TIMESTAMP`, `NULLIF('3', n)` a number β€” or NULL if equal **Examples:** @@ -264,20 +265,27 @@ FROM products -- Converts 0 prices to NULL ``` -**3. Avoid division by zero β€” no longer needed, and measured not to work:** +**3. Guard a zero divisor β€” needed for `%`, not for `/`:** ```sql --- ⚠️ Since 0.24.0 the engine returns NULL for a zero divisor by itself, so write this: +-- Since 0.24.0 the engine returns NULL for a zero divisor of `/` by itself, so write this: SELECT total_sales / total_orders AS avg_order_value FROM sales_summary -- NULL on the rows where total_orders = 0 + +-- `%` is not guarded by itself: NULLIF is the guard +SELECT + total_sales % NULLIF(total_orders, 0) AS remainder +FROM sales_summary +-- NULL on the rows where total_orders = 0 or is missing ``` -> ⚠️ `total_sales / NULLIF(total_orders, 0)` β€” the idiom this section used to recommend β€” throws a -> `null_pointer_exception` in a search on exactly the rows where `total_orders = 0`, and silently -> drops the computed column in an ingest pipeline. Measured on real Elasticsearch, identically -> before and after `0.24.0`. `NULLIF` remains correct everywhere else; see the -> [`/` operator](operators.md) for the division rule. +> `total_sales / NULLIF(total_orders, 0)` β€” the idiom this section used to recommend β€” answers +> `NULL` on the rows where `total_orders` is `0` or missing, in a search and in a computed column, +> and so does `total_sales % NULLIF(total_orders, 0)`. Before `0.24.0` both threw a +> `null_pointer_exception` in a search on exactly those rows, and silently dropped the computed +> column in an ingest pipeline. See the [`/` and `%` operators](operators.md) for the zero-divisor +> rules. **4. Clean data:** ```sql @@ -323,14 +331,15 @@ FROM products failing β€” so the `NULLIF` returned its first argument for every row and the query answered `200` with the wrong values. If you relied on one of these, write the comparison in the type you mean. -- A **string literal against a `date` column** is refused, and the message depends on the literal. - A malformed one is named as such β€” `NULLIF(created_at, 'yesterday')` reports that `'yesterday'` - is not a date for that field's format, checked against the column's mapping exactly as a `WHERE` - comparison is. A **well-formed** one β€” `NULLIF(created_at, '2024-01-15')` β€” is refused too, with - a message saying the comparison is not supported yet: the engine cannot turn a string into a - temporal value inside a script, so accepting it would mean comparing a date with a string, which - silently never matches. Cast the literal instead: - `NULLIF(created_at, CAST('2024-01-15' AS DATE))`. +- A **string literal against a `date` column** is read by what it is compared with, as in every + comparison: `NULLIF(created_at, '2024-01-15')` compares `created_at` with the date 2024-01-15, + and `NULLIF(qty, '3')` compares `qty` with the number 3 β€” in a query and in a computed column + alike (before **0.24.0** the first was refused, and a computed column compared a date with a + string, which never matched). A literal that spells no date is refused, and the message depends + on the literal: a malformed one is named as such β€” `NULLIF(created_at, 'yesterday')` reports + that `'yesterday'` is not a date for that field's format, checked against the column's mapping + exactly as a `WHERE` comparison is β€” and Elasticsearch date math (`'now-1d'`), which no script + can evaluate, is refused as not supported. - `expr2` being NULL is not a match: SQL says the comparison is then UNKNOWN, so the answer is `expr1`, not NULL. @@ -451,7 +460,7 @@ FROM products ### GREATEST -Returns the largest value among the given numeric expressions. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. +Returns the largest value among the given expressions β€” numbers, or dates and timestamps. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. **Syntax:** ```sql @@ -459,10 +468,13 @@ GREATEST(expression1, expression2, ...) ``` **Inputs:** -- One or more numeric expressions to compare +- One or more expressions of one kind: all numeric, or all `DATE` / `TIMESTAMP` / `DATETIME` **Output:** -- Numeric (widest input type), or NULL if every argument is NULL +- Over numbers, the common type of all the arguments, as PostgreSQL types it, and every row has + it: the widest whole type when every argument is a whole number, a `DOUBLE` as soon as one is + fractional (a `REAL` when the fractional ones are all `REAL`); over dates, a `DATE` when every + argument is a `DATE` and a `TIMESTAMP` when they mix; NULL if every argument is NULL **Examples:** @@ -479,16 +491,37 @@ FROM orders -- Never lets the net go below zero ``` +**3. The latest of two dates, and the days since the first:** +```sql +SELECT GREATEST(ordered_on, shipped_on) AS last_step, + GREATEST(ordered_on, shipped_on) - ordered_on AS days_to_last_step +FROM orders +``` + +**4. Per group, over aggregates:** +```sql +SELECT region, GREATEST(MAX(ordered_on), MAX(shipped_on)) AS last_activity +FROM orders +GROUP BY region +``` + **Notes:** - Ignores NULL arguments (PostgreSQL-style NULL handling); returns NULL only if every argument is NULL. -- All arguments should be numeric and have comparable types. +- The arguments are all numeric, or all dates and timestamps, as in every SQL engine; any other + argument, or numbers mixed with dates, is refused by name once the column types are known. +- Dates compare as instants, a `DATE` as the start of its day (UTC); the result is a date like any + other: `GREATEST(d, d2) - d` is a number of days (see [Date arithmetic](operators.md)). +- Over aggregates it is computed per group. In a SELECT, a group missing one of the aggregates' + values answers NULL: Elasticsearch does not run a per-group calculation there (its + `bucket_script` skips a group with a missing metric), as for any per-group calculation. Written + inline in `HAVING`, the group filter reads every aggregate and skips a NULL one. - See also: `LEAST`, `COALESCE`. --- ### LEAST -Returns the smallest value among the given numeric expressions. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. +Returns the smallest value among the given expressions β€” numbers, or dates and timestamps. NULL arguments are ignored (PostgreSQL-style NULL handling); the result is NULL only if every argument is NULL. **Syntax:** ```sql @@ -496,10 +529,13 @@ LEAST(expression1, expression2, ...) ``` **Inputs:** -- One or more numeric expressions to compare +- One or more expressions of one kind: all numeric, or all `DATE` / `TIMESTAMP` / `DATETIME` **Output:** -- Numeric (widest input type), or NULL if every argument is NULL +- Over numbers, the common type of all the arguments, as PostgreSQL types it, and every row has + it: the widest whole type when every argument is a whole number, a `DOUBLE` as soon as one is + fractional (a `REAL` when the fractional ones are all `REAL`); over dates, a `DATE` when every + argument is a `DATE` and a `TIMESTAMP` when they mix; NULL if every argument is NULL **Examples:** @@ -516,9 +552,17 @@ FROM orders -- Never lets the rebate exceed the base price ``` +**3. The earlier of a date and a timestamp:** +```sql +SELECT LEAST(due_on, delivered_at) AS first_event +FROM orders +-- a TIMESTAMP: the arguments mix a DATE and a TIMESTAMP +``` + **Notes:** - Ignores NULL arguments (PostgreSQL-style NULL handling); returns NULL only if every argument is NULL. -- All arguments should be numeric and have comparable types. +- The arguments are all numeric, or all dates and timestamps; see `GREATEST` for dates, and for + `LEAST` over aggregates per group. - See also: `GREATEST`, `COALESCE`. [Back to index](README.md) diff --git a/documentation/sql/functions_date_time.md b/documentation/sql/functions_date_time.md index baecc694f..528928b75 100644 --- a/documentation/sql/functions_date_time.md +++ b/documentation/sql/functions_date_time.md @@ -103,6 +103,25 @@ SELECT CURTIME() AS current_time; ### Date/Time Arithmetic Functions +> **`+` and `-` on dates.** Since `0.24.0` a date or a timestamp can also be moved and subtracted +> with the plain operators, by the rules of the SQL engines: `DATE - DATE` is the calendar days +> between the two dates (`BIGINT`, as in PostgreSQL, DuckDB, Oracle and Snowflake); a difference with +> a `TIMESTAMP` on either side is the time between them in fractional days (`DOUBLE`, as Oracle +> answers for its date with a time of day); `DATE + n` is the date `n` days later (`DATE`, as in +> PostgreSQL, DuckDB, Oracle and BigQuery) β€” a fractional `n` gives a `TIMESTAMP` whose fraction is +> a time of day, as in Oracle; `TIMESTAMP + n` is `n` days later; a number written as a string +> (`'30'`) counts as that number, by elasticsql's own rule (PostgreSQL refuses `DATE + '1'` and +> reads `TIMESTAMP + '1'` as one second). `*`, `/` and `%` on a +> date, two dates added and a number minus a date are refused, as PostgreSQL, DuckDB, Oracle, Trino, +> Snowflake and SQL Server refuse them. See +> [Date arithmetic](operators.md#date-arithmetic--and---with-a-date-timestamp-or-datetime). +> +> ```sql +> SELECT ship_date - order_date AS days_to_ship FROM orders; -- BIGINT +> SELECT due_date + 30 AS reminder FROM invoices; -- DATE +> SELECT * FROM orders WHERE order_date > CURRENT_DATE - 7; +> ``` + #### INTERVAL Literal syntax for time intervals. @@ -346,6 +365,8 @@ TIMESTAMPDIFF(unit, date1, date2) -- MySQL **Direction, by layout:** every one of these databases computes *end βˆ’ start*; the layout decides where the end sits. - The dates first: `date1 - date2`. `DATEDIFF('2025-01-10', '2025-01-01')` is `9`, as in MySQL; `DATE_DIFF('2010-07-07', '2008-12-25', DAY)` is `559`, as in BigQuery. - The unit first: `date2 - date1`. `DATEDIFF(DAY, '2025-01-01', '2025-01-10')` is `9`. +- Between two `DATE`s, `DATEDIFF(date1, date2)` is the same number as the operator `date1 - date2` + (see [Date arithmetic](operators.md#date-arithmetic--and---with-a-date-timestamp-or-datetime)). **Counting, by name:** - `DATEDIFF` and `DATE_DIFF` count the calendar **boundaries crossed**, as SQL Server, Snowflake, Redshift, BigQuery and DuckDB do: one second across a year end is 1 `YEAR`, `1992-09-15` to `1992-11-14` is 2 `MONTH`s, `10:59` to `11:00` is 1 `HOUR`. `DAY` counts calendar days: `2025-01-10T23:30:00Z` and `2025-01-11T00:30:00Z` are 1 day apart. diff --git a/documentation/sql/functions_math.md b/documentation/sql/functions_math.md index 0181f9eb4..4837cf8d0 100644 --- a/documentation/sql/functions_math.md +++ b/documentation/sql/functions_math.md @@ -21,7 +21,8 @@ ABS(x) - `x` - `NUMERIC` **Output:** -- `NUMERIC` +- The type of `x`, as PostgreSQL's `abs`: `ABS` of an `INT` is an `INT` (a whole number), of a + `DOUBLE` a `DOUBLE` **Examples:** ```sql diff --git a/documentation/sql/functions_type_conversion.md b/documentation/sql/functions_type_conversion.md index 93028120d..851989579 100644 --- a/documentation/sql/functions_type_conversion.md +++ b/documentation/sql/functions_type_conversion.md @@ -147,6 +147,10 @@ SELECT CAST(CURRENT_TIMESTAMP AS DATE) AS d; -- a seconds-precision epoch lands in 1970. Multiply by 1000, or use a millisecond epoch. SELECT CAST(1704902400000 AS TIMESTAMP) AS ts; -- Result: 2024-01-10 12:00:00 + +-- The same over an integer or decimal column (created_ms), in a SELECT, in WHERE and in a +-- computed column; a fraction of a millisecond is dropped, toward zero +SELECT CAST(created_ms AS TIMESTAMP) AS created_at FROM events; ``` **Boolean Conversions:** @@ -656,6 +660,7 @@ WHERE denominator != 0; | INT | BOOLEAN | `CAST(1 AS BOOLEAN)` | 0=false, non-zero=true | | DOUBLE | INT | `CAST(123.99 AS INT)` | Truncates decimal | | DOUBLE | VARCHAR | `CAST(123.45 AS VARCHAR)` | Always succeeds | +| BIGINT | TIMESTAMP | `CAST(epoch_ms AS TIMESTAMP)` | Epoch milliseconds, UTC | | DATE | VARCHAR | `CAST(date_col AS VARCHAR)` | Format: 'YYYY-MM-DD' | | DATE | TIMESTAMP | `CAST(date_col AS TIMESTAMP)` | Time set to 00:00:00 | | TIMESTAMP | DATE | `CAST(ts_col AS DATE)` | Drops time component | diff --git a/documentation/sql/known_limitations.md b/documentation/sql/known_limitations.md index 38a1b1bce..e9df7a3c9 100644 --- a/documentation/sql/known_limitations.md +++ b/documentation/sql/known_limitations.md @@ -222,37 +222,32 @@ branch. The restriction is not specific to division, and parenthesising does not | `CAST(a / b AS INTEGER)`, `CAST(a + b AS INTEGER)`, `CAST((a / b) AS INTEGER)` | Parse error | | `FLOOR(a / b)`, `ABS(a / b)`, `COALESCE(a / b, 0)` | Parse error | | `CASE WHEN b != 0 THEN a / b ELSE 0 END` | Parse error | -| `a / NULLIF(b, 0)`, `FLOOR(x)`, `CAST(x AS INTEGER)` | Accepted | +| `(a / b)::INTEGER`, `a / NULLIF(b, 0)`, `FLOOR(x)`, `CAST(x AS INTEGER)` | Accepted | The rule is **directional**: arithmetic *over* a function call is fine, a function call *over* arithmetic is not. -⚠️ **One spelling parses and then ignores the cast.** `(a / b)::INTEGER` is accepted, but the -conversion is discarded: it emits exactly what `a / b` emits, so the result is a DOUBLE even though -the expression reports INTEGER. Do not use it as a workaround for the rejections above β€” it is the -one shape in this family that fails *silently* rather than loudly. It is pre-existing and applies to -any operator (`(a + b)::DOUBLE`, `(d + 1)::INTEGER`). - -The practical consequences are that there is no single-expression way to write a truncated quotient, -and no in-expression guard for `%`: +**The `::` spelling works.** Since `0.24.0`, `(a / b)::INTEGER` applies the cast, truncated toward +zero: `(n / m)::INTEGER` is `3` for 7 / 2 and `-3` for -7 / 2, and NULL for a zero divisor, in a +search and in a computed column. Before `0.24.0` the conversion was discarded and the result stayed +a DOUBLE. This holds for any operator (`(a + b)::BIGINT`), so a truncated quotient is one +expression: ```sql --- instead of CAST(n / m AS INTEGER), compute the quotient into a column and cast THAT column -CREATE TABLE t (n INTEGER, m INTEGER, q DOUBLE SCRIPT AS (n / m)); -SELECT CAST(q AS INTEGER) AS whole FROM t; +-- instead of CAST(n / m AS INTEGER) +SELECT (n / m)::INTEGER AS whole FROM t; ``` -## `%` by zero is not guarded +## `%` by zero is not guarded by itself The `0.24.0` rule that makes `a / 0` yield NULL covers `/` only. With integer operands `a % 0` throws β€” HTTP 400 in a search, and in a computed column the ingest processor's `ignore_failure` swallows it so the column is simply absent. With **floating** operands it produces `NaN`, which Elasticsearch refuses to index, so **the whole document is rejected**. -Because of the restriction above there is no in-expression guard, and `a % NULLIF(b, 0)` throws a -`null_pointer_exception` on exactly the rows the guard is for. Keep a zero divisor out of the data -instead β€” filter it in `WHERE`, or compute the remainder from an already-filtered index. See -[operators](operators.md#-mod). +Guard it with `NULLIF`: `a % NULLIF(b, 0)` answers NULL on the rows where `b` is `0` or missing, in +a search and in a computed column (before `0.24.0` it threw a `null_pointer_exception` on exactly +those rows). See [operators](operators.md#-mod). ## `ORDER BY` over arithmetic on a nullable column diff --git a/documentation/sql/operators.md b/documentation/sql/operators.md index f286ae890..0b3152d94 100644 --- a/documentation/sql/operators.md +++ b/documentation/sql/operators.md @@ -30,10 +30,11 @@ expr1 + expr2 ``` **Inputs:** -- `expr1`, `expr2` - Numeric expressions (`INT`, `DOUBLE`, `DECIMAL`, etc.) +- `expr1`, `expr2` - Numeric expressions (`INT`, `DOUBLE`, `DECIMAL`, etc.), or a `DATE`, + `TIMESTAMP` or `DATETIME` and a number of days (see [Date arithmetic](#date-arithmetic--and---with-a-date-timestamp-or-datetime)) **Output:** -- Numeric type (result type depends on operand types) +- Numeric type (result type depends on operand types); a date or a timestamp for date arithmetic **Examples:** @@ -65,6 +66,15 @@ SELECT 10.5 + 20.3 AS total; -- Mixed types (INT + DOUBLE) SELECT 10 + 20.5 AS total; -- Result: 30.5 (promoted to DOUBLE) + +-- A number and a string column (n INT, s KEYWORD) are a type error, refused by name once the +-- column types are known +SELECT n + s AS x FROM t; +-- Error: Type mismatch: output 'INT' is not compatible with input 'KEYWORD' + +-- To join them as text, convert the number +SELECT CONCAT(CAST(n AS VARCHAR), s) AS x FROM t; +-- Result: '3a' for n = 3, s = 'a' ``` **NULL Handling:** @@ -129,10 +139,14 @@ SELECT 100 + (-50) AS result; -- Result: 50 ``` -**Date Arithmetic:** +**Date Arithmetic** (see [Date arithmetic](#date-arithmetic--and---with-a-date-timestamp-or-datetime)): ```sql --- Date subtraction (days between) -SELECT order_date - ship_date AS days_to_ship +-- Date subtraction: the days between the two dates (BIGINT) +SELECT ship_date - order_date AS days_to_ship +FROM orders; + +-- A date moved back by days (DATE) +SELECT order_date - 7 AS week_before FROM orders; -- With INTERVAL @@ -276,25 +290,21 @@ SELECT CAST(10 AS DOUBLE) / 3 AS result; -- 3.333... > `CREATE TABLE u (n INTEGER, m INTEGER, c DOUBLE SCRIPT AS (n / m))` stored `3.0` for 7/2 before > `0.24.0`; it stores `3.5`. > -> ⚠️ **There is no truncating-division operator, and no single expression that produces one.** -> Other engines spell truncation `DIV` (MySQL) or `//` (DuckDB); this engine has neither β€” and the -> obvious rewrites are **parse errors**, because an arithmetic expression is not accepted as the -> operand of a function, of a `CAST` or of a `CASE` branch: +> ⚠️ **There is no truncating-division operator: write `(a / b)::INTEGER`.** Other engines spell +> truncation `DIV` (MySQL) or `//` (DuckDB); this engine has neither. Since `0.24.0` a `::` cast +> applies to arithmetic and truncates toward zero: `(n / m)::INTEGER` is `3` for 7 / 2 and `-3` for +> -7 / 2, and NULL for a zero divisor, in a search and in a computed column. The other obvious +> rewrites are **parse errors**, because an arithmetic expression is not accepted as the operand of +> a function, of a `CAST` or of a `CASE` branch: > > ```sql > CAST(a / b AS INTEGER) CAST(a + b AS INTEGER) CAST((a / b) AS INTEGER) -- βœ— rejected > FLOOR(a / b) ABS(a / b) COALESCE(a / b, 0) -- βœ— rejected -> FLOOR(x) a / NULLIF(b, 0) -- βœ“ accepted +> (a / b)::INTEGER FLOOR(x) a / NULLIF(b, 0) -- βœ“ accepted > ``` > > The rule is **directional**: `f()` is rejected, ` f(…)` is fine. It is > not specific to division β€” any arithmetic operand is refused β€” and parenthesising does not help. -> To get an integral quotient today, compute it into a column and cast **that** column: -> -> ```sql -> CREATE TABLE t (n INTEGER, m INTEGER, q DOUBLE SCRIPT AS (n / m)); -> SELECT CAST(q AS INTEGER) AS whole FROM t; -- 3 for 7 / 2, truncated toward zero -> ``` > > `%` (MOD) is unaffected by this rule and keeps deriving its type from its operands β€” see the `%` > section below, including what it does with a zero divisor. @@ -319,18 +329,16 @@ division **threw** (a search answered HTTP 400 `arithmetic_exception: / by zero` pipeline silently left the computed column out of the document), while floating division produced `Infinity` β€” which Elasticsearch refuses to index, so **the whole document was rejected**. -> ⚠️ **Two limitations to know about, both of them older than this rule and neither of them fixed -> by it:** +> ⚠️ **One limitation to know about, older than this rule and not fixed by it: `ORDER BY +> ` over nullable columns fails.** A script sort is typed `number`, and Elasticsearch +> rejects a sort script that can return null β€” which any arithmetic over a nullable column can, +> whether from a missing value or from a zero divisor. Measured identically before and after +> `0.24.0`. Sort by a plain column, or by a computed column. > -> - **`ORDER BY ` over nullable columns fails.** A script sort is typed `number`, and -> Elasticsearch rejects a sort script that can return null β€” which any arithmetic over a nullable -> column can, whether from a missing value or from a zero divisor. Measured identically before and -> after `0.24.0`. Sort by a plain column, or by a computed column. -> - **`NULLIF` inside a division fails on exactly the rows it protects.** -> `total / NULLIF(order_count, 0)` throws a `null_pointer_exception` in a search on any row where -> `order_count = 0`, and silently drops the computed column in an ingest pipeline. Measured -> identically before and after `0.24.0`. **Since `0.24.0` you do not need it for division: write -> `total / order_count`.** +> **You do not need `NULLIF` for a division: write `total / order_count`.** It works too: +> `total / NULLIF(order_count, 0)` answers NULL on the rows where `order_count` is `0` or missing, +> in a search and in a computed column. Before `0.24.0` it threw a `null_pointer_exception` in a +> search on exactly those rows, and silently dropped the computed column in an ingest pipeline. ⚠️ **Corrected in `0.24.0`: the other guards this page used to recommend do not parse.** As above, an arithmetic expression is not accepted as the operand of a function, of a `CAST` or of a `CASE` @@ -377,26 +385,23 @@ expr1 % expr2 **Output:** - Integer (remainder of division) -> ⚠️ **A zero divisor is NOT guarded for `%`.** The `0.24.0` rule that turns `a / 0` into NULL -> covers `/` only, and `%` has two distinct failures: +> ⚠️ **A zero divisor is NOT guarded for `%` by itself.** The `0.24.0` rule that turns `a / 0` into +> NULL covers `/` only, and a bare `%` has two distinct failures: > > - **integer operands** β€” `a % 0` throws. A search fails with HTTP 400 > `arithmetic_exception: / by zero`; in a computed column the ingest processor's > `ignore_failure` swallows it and the column is simply **absent** from the indexed document. > - **floating operands** β€” `a % 0` is `NaN`, and Elasticsearch refuses to index a non-finite > number, so **the whole document is rejected** (`document_parsing_exception: [double] supports -> only finite values`). This is the same data-loss failure the `/` guard was shipped to fix, and -> it is still open for `%`. +> only finite values`). This is the same data-loss failure the `/` guard was shipped to fix. > -> ⚠️ **There is no in-expression way to guard it.** `CASE WHEN b != 0 THEN a % b END`, -> `COALESCE(a % b, 0)` and `FLOOR(a % b)` are all **parse errors** β€” an arithmetic expression is -> not accepted as the operand of a function, of a `CAST` or of a `CASE` branch (see the note under -> `/` above; the restriction is not specific to division). And `a % NULLIF(b, 0)` parses but then -> throws `null_pointer_exception` on exactly the rows the guard is for, exactly as it does for -> division. -> -> Until `%` is covered, keep a zero divisor out of the data β€” filter it in `WHERE` -> (`WHERE b <> 0`), or compute the remainder into its own column from an already-filtered index. +> **Guard it with `NULLIF`: write `a % NULLIF(b, 0)`.** It answers NULL on the rows where `b` is +> `0` or missing and the remainder everywhere else, for integer and floating operands alike, in a +> search and in a computed column. Before `0.24.0` it threw a `null_pointer_exception` on exactly +> the rows the guard is for. The other guards one might write are **parse errors**: +> `CASE WHEN b != 0 THEN a % b END`, `COALESCE(a % b, 0)` and `FLOOR(a % b)` β€” an arithmetic +> expression is not accepted as the operand of a function, of a `CAST` or of a `CASE` branch (see +> the note under `/` above; the restriction is not specific to division). **Examples:** @@ -453,6 +458,142 @@ SELECT 10 % -3 AS result; --- +### Date arithmetic: `+` and `-` with a `DATE`, `TIMESTAMP` or `DATETIME` + +**Description:** +Since `0.24.0`, `+` and `-` follow the rules SQL engines apply to dates, in every place a script +runs: a row `SELECT`, a `WHERE`, a per-group calculation over aggregates (`MAX(d) - MIN(d) AS x`), +a `HAVING` over its alias, a computed column (`SCRIPT AS`) and a materialized view. + +| Expression | Result | Type | +|---|---|---| +| `DATE - DATE` | the calendar days between the two dates | `BIGINT` | +| `TIMESTAMP - TIMESTAMP`, or a `DATE` and a `TIMESTAMP` | the milliseconds between the two, in days (`1.5` is a day and a half) | `DOUBLE` | +| `DATE + n`, `DATE - n`, `n` a whole number (`INT`, `BIGINT`, ...) | the date `n` days later / earlier | `DATE` | +| `DATE + n`, `DATE - n`, `n` a fractional number (`DOUBLE`, `REAL`, `DECIMAL`) | `n` days later / earlier, the fraction as a time of day | `TIMESTAMP` | +| `TIMESTAMP + n`, `TIMESTAMP - n`, any number `n` | `n` days later / earlier | `TIMESTAMP` | +| `n + temporal` | the same as `temporal + n` | as above | +| a temporal and a string literal that is a number (`'30'`, `'1.5'`) | the same as with that number (elasticsql's own rule, not PostgreSQL's: see below) | as above | +| a temporal and `NULL` | `NULL` | the type above, for a typed `NULL` (`CAST(NULL AS DATE) - d` is a `BIGINT`) | +| two temporals added, a temporal under `*`, `/` or `%`, a number minus a temporal, a temporal combined with any other string or a boolean | a type error, refused before anything runs | β€” | + +`DATETIME` behaves as `TIMESTAMP`. Days are UTC calendar days: a `DATE` operand is the UTC day it +falls in, so `DATE - DATE` is the same number as `DATEDIFF(date1, date2)`. + +An operand's type is the type it produces: a column's declared type; an aggregate's own type +(`COUNT` is a `BIGINT`, `AVG` a `DOUBLE`, `SUM` the column's numeric type, `MIN`, `MAX`, +`FIRST_VALUE` and `LAST_VALUE` the column's type); `CASE`, `COALESCE`, `NULLIF`, `GREATEST` and +`LEAST` the common type of their branches or arguments. So `MAX(order_date) - AVG(delay)` is a +`TIMESTAMP`, `COUNT(order_date) - 1` a number, and +`CASE WHEN express THEN ship_date ELSE due_date END - order_date` and +`GREATEST(ship_date, due_date) - order_date` numbers of days (`GREATEST` and `LEAST` compare dates +and timestamps too: see [GREATEST](functions_conditional.md#greatest)). + +**The engines this follows:** +- PostgreSQL, DuckDB, Oracle and Snowflake subtract two dates into a number of days; Oracle, whose + `DATE` carries a time of day, answers fractional days. +- PostgreSQL, DuckDB, Oracle and BigQuery add a number of days to a date; Oracle turns the fraction + of `DATE + 1.5` into a time of day. +- PostgreSQL, DuckDB, Oracle, Trino, Snowflake and SQL Server refuse the other combinations. +- A number written as a string beside a date or a timestamp (`order_date + '30'`) is elasticsql's + own rule, not PostgreSQL's: PostgreSQL refuses `order_date + '1'` + (`operator is not unique: date + unknown`), reads the `'1'` of `order_date - '1'` as a date and + refuses it (`invalid input syntax for type date`), and reads `created_at + '1'` as one second. + +**Examples:** +```sql +-- Days between two dates +SELECT ship_date - order_date AS days_to_ship FROM orders; +-- 2024-01-31 -> 2024-02-01 is 1; 2024-02-28 -> 2024-03-01 is 2 (2024 is a leap year) + +-- A date moved by days +SELECT due_date + 30 AS reminder FROM invoices; -- DATE +SELECT due_date + 1.5 AS reminder_at FROM invoices; -- TIMESTAMP, 12:00 on the next day + +-- Fractional days between two instants +SELECT closed_at - opened_at AS days_open FROM tickets; -- 1.25 is a day and six hours + +-- A filter relative to today +SELECT * FROM orders WHERE order_date > CURRENT_DATE - 7; + +-- Per group, and in HAVING through the alias +SELECT customer, MAX(order_date) - MIN(order_date) AS span +FROM orders GROUP BY customer HAVING span > 30; + +-- A computed column +CREATE TABLE tickets ( + id KEYWORD, opened_at TIMESTAMP, closed_at TIMESTAMP, + days_open DOUBLE SCRIPT AS (closed_at - opened_at), + PRIMARY KEY (id) +); +``` + +**Refused combinations** are refused by name, with the operator, the two operand types and the +remedy: +```sql +SELECT order_date * 2 FROM orders; +-- Type mismatch: operator * cannot be applied to DATE and BIGINT in expression: order_date * 2; +-- subtract two dates for the days between them, or add or subtract a number of days +``` + +**Notes:** +- A temporal operand is recognised by its **declared** type: Elasticsearch stores `DATE`, + `TIMESTAMP` and `DATETIME` alike as a `date`, and the declaration is what tells a date from a + timestamp. A `TIME` operand is not covered by these rules. +- A `date` field elasticsql did not declare β€” an index created by a bulk load, Logstash or any other + client, or a field a bulk load added to a table elasticsql created β€” is a `TIMESTAMP`: `DESCRIBE + TABLE` reports it so, a `CREATE TABLE ... AS SELECT` declares it so, and it follows the + `TIMESTAMP` rules (`created_at + 1` keeps the time of day, `closed_at - opened_at` is + fractional). A column declared `DATE` through elasticsql stays a `DATE`. +- A string literal is read by what it is combined or compared with, as PostgreSQL reads one: + `order_date - '2024-01-01'` is the days between two dates, `qty + '1'` adds the number 1, and in + a comparison β€” `=`, `<>`, `<`, `<=`, `>`, `>=`, `BETWEEN`, `IN`, `CASE x WHEN v`, `NULLIF`, a + `CASE` condition β€” a literal beside a number is the number it spells, beside a `DATE` the date + it spells (the date of a date-time literal), beside a `TIMESTAMP` the instant it spells + (`'2024-01-31'` is its midnight), and beside a `BOOLEAN` the boolean it spells. Beside a + whole-number type (`TINYINT`, `SMALLINT`, `INT`, `BIGINT`) only a whole number is read: `'1.5'` + and `'1.0'` are no integer, so `COALESCE(qty, 0) = '1.5'` or `qty + '1.0'` over an integer `qty` + is refused by name, as PostgreSQL refuses it (`invalid input syntax for type integer`); beside + `DOUBLE`, `REAL` or `DECIMAL`, any number is read. (A `WHERE` comparison with a bare integer + column, `WHERE qty = '1.5'`, is the exception: it runs as a query on the column's mapping, and + Elasticsearch reads the literal.) A `DATE` and a `TIMESTAMP` compare, the `DATE` being the + instant its day starts at. Any other literal is text, and comparing it with a number or a date + is refused by name. A number written as a string beside a date or a timestamp in `+` or `-` + (`order_date + '30'`) is not read as PostgreSQL reads it: elasticsql counts it as that number of + days, whatever its fraction, which PostgreSQL does not (see "The engines this follows" above). +- An index pattern or a comma list (`FROM logs-*`, `FROM orders_2024, orders_2025`) answers like a + single index: every field its indices map alike has its type, read once per pattern from the + merged mapping. ⚠️ A field two of them map differently (a `date` in one index, a `keyword` in + another) has no type: nothing is refused on it and Elasticsearch answers for it β€” per group, + `MAX(order_date) - MIN(order_date)` over such a field answers milliseconds rather than days. Every + other field of the pattern keeps its type. +- Type errors are checked once the column types are known, when the statement runs and the mapping + is read, and before anything is sent to Elasticsearch: `WHERE due_date + 7 > CURRENT_DATE` runs, + `SELECT order_date * 2` is refused. With no type to read (a field the indices of a pattern map + differently, a mapping that cannot be read), nothing is refused on types. +- In a per-group calculation, a date result comes back as the same number a date aggregate does + (`MAX(order_date)`): milliseconds since the epoch. `CURRENT_DATE` and `NOW()` there are the time + the statement runs (`CURRENT_DATE - MAX(order_date)`); a materialized view refuses them. +- An explicit cast applies to the result, and a `NULL` stays `NULL`: + `(ship_date - order_date)::TIMESTAMP` converts the number of days as any number is converted to a + `TIMESTAMP` β€” as milliseconds since the epoch β€” and `(quantity + ratio)::BIGINT` truncates, as + `CAST(3.25 AS BIGINT)` does. +- `INTERVAL` arithmetic (`order_date - INTERVAL 7 DAY`) and `DATEDIFF` / `TIMESTAMPDIFF` are + unchanged. +- ⚠️ On Elasticsearch 6.8, a `date` column compared with another `date` column + (`WHERE ship_date > order_date`), or with a date function of one + (`WHERE ship_date > DATE_TRUNC(order_date, MONTH)`), fails with a 400 (`failed to parse date + field`); date arithmetic on one side runs: `WHERE ship_date > order_date + 0`. Elasticsearch 8 + answers all three. + +**Before `0.24.0`:** `MAX(d) - MIN(d)` answered milliseconds and was typed `NUMERIC`; `+` and `-` +with a date failed in Elasticsearch at row level β€” except a date plus a string, which concatenated +the two β€” and per group `MAX(d) + 1` failed while `MAX(d) + 1.5` added 1.5 milliseconds; +`CURRENT_DATE - 7` was refused when the statement was parsed; and a computed column over date +arithmetic stored epoch-millisecond arithmetic or rejected the document at ingest. + +--- + ## Comparison Operators ### Operator: `=` @@ -485,6 +626,14 @@ SELECT * FROM emp WHERE department = 'IT'; -- Compare columns SELECT * FROM orders WHERE customer_id = shipping_customer_id; + +-- Columns of incompatible types (id KEYWORD, n INT) are refused by name once the column types +-- are known, rather than answering no rows +SELECT * FROM t WHERE id = n; +-- Error: Type mismatch: 'KEYWORD' is not compatible with 'INT' in expression: id = n + +-- Cast one side to the other's type +SELECT * FROM t WHERE id = CAST(n AS VARCHAR); ``` **String Comparison:** diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala new file mode 100644 index 000000000..7e661b3aa --- /dev/null +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/BucketScriptParams.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.bridge + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.node.{ArrayNode, ObjectNode} +import com.sksamuel.elastic4s.searches.SearchRequest +import com.sksamuel.elastic4s.searches.aggs.{AbstractAggregation, Aggregation} +import com.sksamuel.elastic4s.searches.aggs.pipeline.BucketScriptPipelineAgg + +import scala.jdk.CollectionConverters._ + +/** The PARAMS of a `bucket_script`'s script, which the elastic4s body builder drops. + * + * πŸ”΄ elastic4s writes a `bucket_script` as `"script": ""` -- the bare source string + * (`BucketScriptPipelineAggBuilder`, the same in elastic4s 6.7, 7.17, 8.18 and 9.0) -- while a + * `bucket_selector` gets the whole script object. So a per-group calculation reading the request + * clock, `CURRENT_DATE - MAX(d)` or `NOW() - MAX(ts)`, was sent WITHOUT the `__now__` its script + * reads, and Elasticsearch failed it; the same clock in a HAVING worked. MEASURED on 8.18.3. + * + * The script is sent as the object Elasticsearch documents -- `source`, `lang`, `params` -- for + * exactly the `bucket_script`s that carry params, so the source stays the SAME text from one + * request to the next and Elasticsearch compiles it once: inlining the clock instead would compile + * a new script on every request, against the cluster's compilation rate limit. Every other body is + * returned untouched, unparsed. + */ +object BucketScriptParams { + + private val mapper = new ObjectMapper() + + /** The params of every `bucket_script` of `aggs` that carries some, by aggregation name, at any + * depth. + */ + private def withParams(aggs: Seq[AbstractAggregation]): Map[String, Map[String, Any]] = + aggs.flatMap { + case script: BucketScriptPipelineAgg if script.script.params.nonEmpty => + Seq(script.name -> script.script.params) + case parent: Aggregation => withParams(parent.subaggs).toSeq + case _ => Nil + }.toMap + + /** `body`, the JSON of `search`, with the params of its `bucket_script`s bound. */ + def bind(body: String, search: SearchRequest): String = { + val params = withParams(search.aggs) + if (params.isEmpty) body + else { + val root = mapper.readTree(body) + bind(root, params) + mapper.writeValueAsString(root) + } + } + + private def bind(node: JsonNode, params: Map[String, Map[String, Any]]): Unit = + node match { + case obj: ObjectNode => + obj.fields().asScala.toList.foreach { entry => + (params.get(entry.getKey), entry.getValue) match { + case (Some(values), agg: ObjectNode) if agg.has("bucket_script") => + agg.get("bucket_script") match { + case bucketScript: ObjectNode if bucketScript.path("script").isTextual => + val script = mapper.createObjectNode() + script.put("source", bucketScript.get("script").asText()) + script.put("lang", "painless") + val bound = script.putObject("params") + values.foreach { case (name, value) => + bound.set[JsonNode](name, mapper.valueToTree[JsonNode](value)) + } + bucketScript.set[JsonNode]("script", script) + case _ => + } + case _ => + } + bind(entry.getValue, params) + } + case array: ArrayNode => array.elements().asScala.foreach(bind(_, params)) + case _ => + } +} diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala index 5eccb1ea1..7eacc6332 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.`type`.SQLTemporal import app.softnetwork.elastic.sql.query.{ Asc, @@ -162,7 +162,8 @@ object ElasticAggregation { allAggregations: Map[String, SQLAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): ElasticAggregation = { import sqlAgg._ val sourceField = identifier.path @@ -219,14 +220,14 @@ object ElasticAggregation { buildScript: (String, Script) => Aggregation ): Aggregation = { if (transformFuncs.nonEmpty) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) } else { aggType match { case th: WindowFunction if th.shouldBeScripted => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val scriptSrc = th.identifier.painless(Some(context)) val script = now(Script(s"$context$scriptSrc").lang("painless")) buildScript(aggName, script) @@ -471,7 +472,8 @@ object ElasticAggregation { allElasticAggregations: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[Aggregation] = { for (tree <- buckets) yield { val treeNodes = @@ -488,7 +490,7 @@ object ElasticAggregation { val aggScript = if (!bucket.isBucketScript && bucket.shouldBeScripted) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = bucket.painless(Some(context)) Some(now(Script(s"$context$painless").lang("painless"))) } else { @@ -716,7 +718,8 @@ object ElasticAggregation { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Option[Aggregation] = request.having.flatMap(_.criteria) match { case Some(criteria) if request.wholeTableHaving && aggs.nonEmpty => diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala index 6eca023d4..a6985f06a 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticBridge.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.operator.AND import app.softnetwork.elastic.sql.query.{ BetweenExpr, @@ -51,7 +51,8 @@ case class ElasticBridge(filter: ElasticFilter) { currentQuery: Option[ElasticBoolQuery] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { filter match { case boolQuery: ElasticBoolQuery => diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala index efb14be16..b8610990c 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -16,7 +16,7 @@ package app.softnetwork.elastic.sql.bridge -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.query.Criteria import com.sksamuel.elastic4s.searches.queries.Query @@ -24,7 +24,8 @@ case class ElasticCriteria(criteria: Criteria) { def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { val query = criteria.boolQuery.copy(group = group) query diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index 6782773f9..49dcc4e28 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -49,5 +49,10 @@ case class ElasticSearchRequest( def hasTransformExtendedStats: Boolean = SearchBodySerializer.hasTransformExtendedStats(search) def query: String = - serializer.serialize(search).replace("\"version\":true,", "") /*FIXME*/ + // the params of a per-group calculation's script, which the body builder drops: the request + // clock `CURRENT_DATE - MAX(d)` reads (`BucketScriptParams`) + BucketScriptParams.bind( + serializer.serialize(search).replace("\"version\":true,", ""), /*FIXME*/ + search + ) } diff --git a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala index f8dc25247..daadf6e07 100644 --- a/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala +++ b/es6/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -75,7 +75,8 @@ package object bridge { innerHitsName: String )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = { val having: Option[Query] = request.having.flatMap(_.criteria) match { @@ -154,7 +155,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Option[FilterAggregation] = request.having.flatMap(_.criteria) match { case Some(f) => @@ -182,7 +184,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[AbstractAggregation] = { val notNestedAggregations = aggregations.filterNot(_.nested) @@ -255,7 +258,8 @@ package object bridge { references: Seq[ElasticAggregation] )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Seq[NestedAggregation] = { // Group nested aggregations by their nested path val nestedAggregations: Map[String, Seq[ElasticAggregation]] = aggregations @@ -468,10 +472,16 @@ package object bridge { // SearchBodySerializer in implicit scope of this conversion; with none in scope the default // argument applies (the one-argument elastic4s builder, refusing a transform-bearing // extended_stats). Same defaulted-implicit pattern as `contextType`. + // + // `target` is the same seam for SCRIPTS: the Elasticsearch major the client module talks to + // (`PainlessTarget`), which every conversion below hands to the `PainlessContext`s it creates. + // With none in scope a script renders as for the majors from 7 on; the ES 6 client modules + // give 6, whose query scripts compare a `date` doc value differently. implicit def requestToElasticSearchRequest(request: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): ElasticSearchRequest = ElasticSearchRequest( request.sql, @@ -530,7 +540,8 @@ package object bridge { request: SingleSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): SearchRequest = { import request._ @@ -593,7 +604,7 @@ package object bridge { case Nil => _search case _ => _search scriptfields scriptFields.map { field => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = field.painless(Some(context)) scriptField( field.scriptName, @@ -614,7 +625,7 @@ package object bridge { case Some(o) if aggregates.isEmpty && buckets.isEmpty => _search sortBy o.sorts.map { sort => if (sort.isScriptSort) { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val painless = sort.field.painless(Some(context)) val painlessScript = s"$context$painless" val script = @@ -680,7 +691,8 @@ package object bridge { request: MultiSearch )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): MultiSearchRequest = { // Story 22.6 β€” a NAMED backstop, never a silent `UNION ALL`. Only `UNION ALL` is an // Elasticsearch `_msearch`; every other set operator needs the relational engine and is @@ -717,9 +729,10 @@ package object bridge { private[bridge] def scriptQueryOf(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType + contextType: PainlessContextType, + target: PainlessTarget ): Query = { - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = criteria.painless(Some(context)) scriptQuery( now(Script(script = s"$context$script").lang("painless").scriptType("source")) @@ -733,7 +746,8 @@ package object bridge { implicit def expressionToQuery(expression: GenericExpression)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import expression._ if (isAggregation) @@ -958,7 +972,7 @@ package object bridge { case _ => scriptQueryOf(expression) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -969,7 +983,7 @@ package object bridge { ) } case _ => - val context = PainlessContext(context = contextType) + val context = PainlessContext(context = contextType, target = target) val script = painless(Some(context)) scriptQuery( now( @@ -1013,7 +1027,8 @@ package object bridge { implicit def inToQuery[R, T <: Value[R]](in: InExpr[R, T])(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { if (requiresScript(in.identifier)) return scriptQueryOf(in) import in._ @@ -1038,7 +1053,8 @@ package object bridge { between: BetweenExpr )(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { import between._ if (requiresScript(identifier)) return scriptQueryOf(between) @@ -1191,14 +1207,16 @@ package object bridge { implicit def criteriaToQuery(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): Query = { ElasticCriteria(criteria).asQuery() } implicit def criteriaToNode(criteria: Criteria)(implicit timestamp: Long, - contextType: PainlessContextType = PainlessContextType.Query + contextType: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default ): JsonNode = { queryToJson(criteriaToQuery(criteria)) } @@ -1217,7 +1235,8 @@ package object bridge { )(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query, - serializer: SearchBodySerializer = SearchBodySerializer.Default + serializer: SearchBodySerializer = SearchBodySerializer.Default, + target: PainlessTarget = PainlessTarget.Default ): Seq[ElasticAggregation] = { import query._ statement @@ -1257,7 +1276,10 @@ package object bridge { aggregation.copy( sources = l.sources, query = Some( - serializer.serialize(body).replace("\"version\":true,", "") /*FIXME*/ + BucketScriptParams.bind( + serializer.serialize(body).replace("\"version\":true,", ""), /*FIXME*/ + body + ) ) ) }) diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala new file mode 100644 index 000000000..f6927d6e9 --- /dev/null +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/BucketScriptParamsSpec.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.query._ +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime +import scala.jdk.CollectionConverters._ + +/** A per-group calculation that reads the request clock (`CURRENT_DATE - MAX(d)`, `NOW() - + * MAX(ts)`) is sent WITH the clock: its `bucket_script` carries the script object -- `source`, + * `lang`, `params.__now__` -- where the elastic4s builder writes the bare source string and drops + * the params (`BucketScriptParams`). Every other `bucket_script` is sent as it was. + * + * The clock is a PARAM, not text spliced into the source: the source is the same from one request + * to the next, so Elasticsearch compiles it once. + */ +class BucketScriptParamsSpec extends AnyFlatSpec with Matchers { + + import scala.language.implicitConversions + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + implicit def timestamp: Long = now + + implicit def sqlQueryToRequest(sqlQuery: SelectStatement): ElasticSearchRequest = + sqlQuery.statement match { + case Some(value: SingleSearch) => value.copy(score = sqlQuery.score) + case other => throw new IllegalArgumentException(s"Not a single search: $other") + } + + private val mapper = new ObjectMapper() + + private def bucketScripts(sql: String): Seq[JsonNode] = { + val request: ElasticSearchRequest = SelectStatement(sql) + def all(node: JsonNode): Seq[JsonNode] = + Option(node.get("bucket_script")).toSeq ++ node.elements().asScala.toSeq.flatMap(all) + all(mapper.readTree(request.query)) + } + + "a per-group calculation reading the clock" should "send the clock as a script param" in { + Seq( + "SELECT g, CURRENT_DATE - MAX(d) AS x FROM t GROUP BY g", + "SELECT g, NOW() - MAX(ts) AS x FROM t GROUP BY g", + "SELECT g, DATEDIFF(CURRENT_DATE, MAX(d)) AS x FROM t GROUP BY g" + ).foreach { sql => + withClue(s"[$sql] ") { + val scripts = bucketScripts(sql) + scripts should have size 1 + val script = scripts.head.get("script") + script.isObject shouldBe true + script.get("source").asText() should include("params.__now__") + script.get("lang").asText() shouldBe "painless" + script.get("params").get("__now__").asLong() shouldBe now + } + } + } + + "a per-group calculation without the clock" should "be sent as it always was" in { + val scripts = bucketScripts("SELECT g, MAX(d) - MIN(d) AS x FROM t GROUP BY g") + scripts should have size 1 + scripts.head.get("script").isTextual shouldBe true + } +} diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala new file mode 100644 index 000000000..e385013b6 --- /dev/null +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/PainlessTargetSpec.scala @@ -0,0 +1,99 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.ZonedDateTime + +/** The Elasticsearch major the client module puts in implicit scope (`PainlessTarget`) reaches + * EVERY query script the bridge's conversions create: a WHERE, a script field, an aggregation over + * an expression. + * + * The probe is a comparison between a `date` doc value and another date, which Elasticsearch 6.8 + * alone needs normalised to UTC: with 6 in scope the body carries the normalisation, with any + * other major -- or none -- it is the very body no target produces. + */ +class PainlessTargetSpec extends AnyFlatSpec with Matchers { + + private val now: Long = ZonedDateTime.parse("2025-12-31T00:00:00Z").toInstant.toEpochMilli + + private val normalisation = ".withZoneSameInstant(ZoneId.of('Z'))" + + private val schema = Parser( + "CREATE TABLE t (id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, n INT, PRIMARY KEY (id))" + ) match { + case Right(ct: CreateTable) => ct.schema + case other => fail(s"$other") + } + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(ss: SingleSearch) => + val r = ss.update(Some(schema)) + r.validateResolved() shouldBe Right(()) + r + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + /** The body `requestToElasticSearchRequest` sends. (`sqlQueryToAggregations`, the other door, + * parses its statement from text: no schema, so no type to normalise a comparison by.) + */ + private def body(sql: String, target: PainlessTarget): String = + requestToElasticSearchRequest(resolved(sql))( + now, + PainlessContextType.Query, + SearchBodySerializer.Default, + target + ).query + + private val venues: Seq[String] = Seq( + // a WHERE: a script query, alone and inside a bool + "SELECT id FROM t WHERE CAST(ts AS DATE) = d", + "SELECT id FROM t WHERE CURRENT_DATE > d AND n > 1", + // a script field + "SELECT id, CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END AS c FROM t", + // an aggregation over an expression + "SELECT SUM(CASE WHEN CURRENT_DATE > d THEN 1 ELSE 0 END) AS s FROM t", + // the WHERE of an aggregate query + "SELECT g, COUNT(*) AS c FROM t WHERE CAST(ts AS DATE) = d GROUP BY g" + ) + + "a script comparing a date doc value" should + "carry the UTC normalisation when the client module gives Elasticsearch 6" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + body(sql, PainlessTarget(6)) should include(normalisation) + } + } + } + + it should "be the body no target produces for Elasticsearch 7, 8 and 9" in { + venues.foreach { sql => + withClue(s"[$sql] ") { + val default = body(sql, PainlessTarget.Default) + default should not include normalisation + Seq(7, 8, 9).foreach(major => body(sql, PainlessTarget(major)) shouldBe default) + body(sql, PainlessTarget(6)).replace(normalisation, "") shouldBe default + } + } + } +} diff --git a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index 6f31f8661..136252270 100644 --- a/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -2509,7 +2509,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | "c": { | "script": { | "lang": "painless", - | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate().plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value.toLocalDate()); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", + | "source": "def param1 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(params.__now__), ZoneId.of('Z')).toLocalDate().minus(7, ChronoUnit.DAYS); def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); def param5 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).plus(2, ChronoUnit.DAYS)); def param6 = (doc['createdAt'].size() == 0 ? null : doc['createdAt'].value); param1 != null && param2 != null && param1.isEqual(param2) ? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6", | "params": { | "__now__": 1767139200000 | } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala index 5fdb8ecc4..22bfe40f7 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchApi.scala @@ -25,7 +25,7 @@ import app.softnetwork.elastic.client.{ } import com.fasterxml.jackson.databind.JsonNode import app.softnetwork.elastic.client.result.ElasticResult -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.bridge.{ElasticSearchRequest, SearchBodySerializer} import app.softnetwork.elastic.sql.query.SingleSearch import io.searchbox.core.{MultiSearch, Search, SearchResult} @@ -71,6 +71,16 @@ trait JestSearchApi extends SearchApi with JestClientHelpers { */ implicit def searchBodySerializer: SearchBodySerializer = JestSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * Elasticsearch 6.8 hands a query script a `date` doc value as a `JodaCompatibleZonedDateTime`, + * so a comparison of one with any other date is normalised to UTC first. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + JestSearchBodySerializer.ElasticsearchMajor + ) + private[client] implicit def singleSearchToJsonQuery( sqlSearch: SingleSearch )(implicit diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala index aaae20f83..86e511a0e 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestSearchBodySerializer.scala @@ -33,7 +33,10 @@ import com.sksamuel.elastic4s.searches.SearchRequest */ object JestSearchBodySerializer extends SearchBodySerializer { - val ElasticsearchMajor: Int = 6 + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = JestClientBuildInfo.elasticMajor /** The named error a transform-bearing extended_stats is refused with on this module. */ val TransformExtendedStatsUnsupported: String = diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientDateArithmeticSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientDateArithmeticSpec.scala new file mode 100644 index 000000000..a646db785 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 7031b951c..6789fcbf0 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} +import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, PainlessTarget, Value} import app.softnetwork.elastic.sql.query.{SQLAggregation, SingleSearch} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.policy.{EnrichPolicy, EnrichPolicyTask} @@ -1052,6 +1052,16 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel */ implicit def searchBodySerializer: SearchBodySerializer = RestHighLevelClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * Elasticsearch 6.8 hands a query script a `date` doc value as a `JodaCompatibleZonedDateTime`, + * so a comparison of one with any other date is normalised to UTC first. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + RestHighLevelClientSearchBodySerializer.ElasticsearchMajor + ) + override implicit def singleSearchToJsonQuery(sqlSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala index 285ae7f14..5a4c177ee 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala @@ -33,7 +33,10 @@ import com.sksamuel.elastic4s.searches.SearchRequest */ object RestHighLevelClientSearchBodySerializer extends SearchBodySerializer { - val ElasticsearchMajor: Int = 6 + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = RestClientBuildInfo.elasticMajor /** The named error a transform-bearing extended_stats is refused with on this module. */ val TransformExtendedStatsUnsupported: String = diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala new file mode 100644 index 000000000..2dde42545 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index 0092c4d47..7e9b7ab31 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -23,7 +23,7 @@ import app.softnetwork.elastic.client._ import app.softnetwork.elastic.client.bulk._ import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticResult, ElasticSuccess} import app.softnetwork.elastic.client.scroll._ -import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, Value} +import app.softnetwork.elastic.sql.{ObjectValue, PainlessContextType, PainlessTarget, Value} import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.policy.{ EnrichPolicy, @@ -1104,6 +1104,15 @@ trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientHel */ implicit def searchBodySerializer: SearchBodySerializer = RestHighLevelClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + RestHighLevelClientSearchBodySerializer.ElasticsearchMajor + ) + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala index 3bc5439ec..8cb7fd492 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientSearchBodySerializer.scala @@ -49,7 +49,10 @@ import com.sksamuel.elastic4s.requests.searches.aggs.{AbstractAggregation, Aggre */ object RestHighLevelClientSearchBodySerializer extends SearchBodySerializer { - val ElasticsearchMajor: Int = 7 + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = RestClientBuildInfo.elasticMajor /** The request with every [[ScriptedExtendedStatsAggregation]] replaced by the * `ExtendedStatsAggregation` it wraps β€” at the root and at every depth (the windowed bind sits diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala new file mode 100644 index 000000000..2dde42545 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 9fac19776..befb890fe 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -38,7 +38,7 @@ import app.softnetwork.elastic.sql.policy.{ EnrichPolicyTaskStatus, EnrichPolicyType } -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.transform.{ Delay, @@ -1034,6 +1034,15 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { */ implicit def searchBodySerializer: SearchBodySerializer = JavaClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + JavaClientSearchBodySerializer.ElasticsearchMajor + ) + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala index 6ff2d46f6..25399cd4d 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala @@ -53,6 +53,11 @@ import com.sksamuel.elastic4s.requests.searches.{ */ object JavaClientSearchBodySerializer extends SearchBodySerializer { + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = JavaClientBuildInfo.elasticMajor + private val scriptedExtendedStats: PartialFunction[AbstractAggregation, XContentBuilder] = { case agg: ScriptedExtendedStatsAggregation => extendedStatsWithScript(agg) } diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala new file mode 100644 index 000000000..de5ed4607 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 09681fda3..30c96e077 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -37,7 +37,7 @@ import app.softnetwork.elastic.sql.policy.{ EnrichPolicyTaskStatus, EnrichPolicyType } -import app.softnetwork.elastic.sql.PainlessContextType +import app.softnetwork.elastic.sql.{PainlessContextType, PainlessTarget} import app.softnetwork.elastic.sql.schema.TableAlias import app.softnetwork.elastic.sql.serialization._ import app.softnetwork.elastic.sql.transform.{ @@ -1034,6 +1034,15 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { */ implicit def searchBodySerializer: SearchBodySerializer = JavaClientSearchBodySerializer + /** The Elasticsearch major every script made from this trait is rendered for (implicit scope of + * the bridge's conversions, as [[searchBodySerializer]] is): see `PainlessTarget`. + * + * A `date` doc value is a `ZonedDateTime` here: scripts render as they always have. + */ + implicit def painlessTarget: PainlessTarget = PainlessTarget( + JavaClientSearchBodySerializer.ElasticsearchMajor + ) + override implicit def singleSearchToJsonQuery(singleSearch: SingleSearch)(implicit timestamp: Long, contextType: PainlessContextType = PainlessContextType.Query diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala index 6ff2d46f6..25399cd4d 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientSearchBodySerializer.scala @@ -53,6 +53,11 @@ import com.sksamuel.elastic4s.requests.searches.{ */ object JavaClientSearchBodySerializer extends SearchBodySerializer { + /** The Elasticsearch major this module is built against, generated from the build's + * `elasticSearchVersion` (never written by hand). + */ + val ElasticsearchMajor: Int = JavaClientBuildInfo.elasticMajor + private val scriptedExtendedStats: PartialFunction[AbstractAggregation, XContentBuilder] = { case agg: ScriptedExtendedStatsAggregation => extendedStatsWithScript(agg) } diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala new file mode 100644 index 000000000..de5ed4607 --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientDateArithmeticSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientDateArithmeticSpec extends DateArithmeticSpec diff --git a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala index 9543ac0eb..584ad3521 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/schema/package.scala @@ -75,8 +75,29 @@ package object schema { } object IndexField { + + /** The type of a field elasticsql never DECLARED -- no `_meta` `data_type` -- read off its + * Elasticsearch mapping type: an index created by a bulk load or Logstash, or a field a bulk + * load added later to a table elasticsql created (dynamic mapping). + * + * πŸ”΄ THE declaration seam (the lead's ruling of 2026-10-05): such a `date` field holds + * INSTANTS + * -- Elasticsearch stores every temporal as one, and nobody said it was a calendar date -- so + * it is a TIMESTAMP, everywhere: DESCRIBE, the type a CTAS declares, comparisons, functions + * and date arithmetic all read it from here, and an `ALTER TABLE` that writes the table's + * `_meta` declares it TIMESTAMP, keeping its time. A field elasticsql DECLARED keeps its + * declaration, a DATE included. + * + * ⚠️ Deliberately NOT in `SQLTypes.apply`: that maps a TYPE NAME, the declared `DATE` as well + * as the mapping's `date`, and story 21.5 learned there that conflating the two rewrites the + * declaration of every DATE column (`ColumnMetaDiffSpec`). Only this seam knows that no + * declaration exists. + */ + private[schema] def undeclaredType(mappingType: String): String = + if (mappingType == "date") "timestamp" else mappingType + def apply(name: String, node: JsonNode, _meta: Option[ObjectValue]): IndexField = { - val tpe = _meta + val declaredType = _meta .flatMap { case m: ObjectValue => m.value.get("data_type") match { @@ -85,7 +106,10 @@ package object schema { } case _ => None } - .getOrElse(Option(node.get("type")).map(_.asText()).getOrElse("object")) + val tpe = + declaredType.getOrElse( + undeclaredType(Option(node.get("type")).map(_.asText()).getOrElse("object")) + ) val nullValue = Option(node.get("null_value")) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala index a93f09664..469691ab7 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/aggregate/package.scala @@ -39,6 +39,55 @@ package object aggregate { } + object AggregateFunction { + + /** The type an AGGREGATE produces -- the metric Elasticsearch computes for a group -- as + * opposed to the type of the column it aggregates, which is what the aggregate identifier's + * `out` reports. + * + * πŸ”΄ ONE derivation for every consumer (#292): the date-arithmetic rules (`COUNT(d) - 1` is a + * number, never a date), the coercion a per-group calculation applies to a metric + * (`params.count_d` is a count, so it is never read as an instant), and the type rules that + * compare a metric once the column types are known (`HAVING COUNT(d) > 1`). Reading the + * column's type instead is what rendered `params.count_d.toInstant()`. + * + * - `COUNT` counts: BIGINT, whatever it counts. + * - `AVG` averages: DOUBLE. + * - `SUM` adds: the summed column's numeric type (whole stays whole, fractional stays + * fractional); a sum of anything else is the DOUBLE Elasticsearch answers -- a sum of + * dates is not a date. + * - `MIN`, `MAX`, `FIRST_VALUE`, `LAST_VALUE` return one of the values they aggregate: the + * REPORTED type of the aggregated chain, so a DATE stays a DATE and `MAX(CAST(ts AS + * DATE))` is a DATE. + * - every other aggregate answers its own declared type when it has one (STDDEV and the + * percentiles are DOUBLE, the ranking windows BIGINT), else the aggregated chain's. + * + * `None` when `aggregate` aggregates nothing. + */ + def outputTypeOf(aggregate: Identifier): Option[SQLType] = + aggregate.aggregateFunction.map { function => + def aggregated: SQLType = + aggregate + .withFunctions(aggregate.functions.filterNot(_.isInstanceOf[AggregateFunction])) + .reportedType + function match { + case COUNT | _: CountAgg => SQLTypes.BigInt + case AVG | _: AvgAgg => SQLTypes.Double + case SUM | _: SumAgg => + aggregated match { + case whole @ (SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | + SQLTypes.BigInt) => + whole + case fractional @ (SQLTypes.Real | SQLTypes.Double) => fractional + case _ => SQLTypes.Double + } + case MIN | MAX | _: MinAgg | _: MaxAgg | _: FirstValue | _: LastValue => aggregated + case other => + Some(other.out).filterNot(t => t.isUnknown || t == SQLTypes.Null).getOrElse(aggregated) + } + } + } + case object COUNT extends Expr("COUNT") with AggregateFunction with Window case object MIN extends Expr("MIN") with AggregateFunction with Window diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala index 4f3f8f1dd..24dec2536 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/cond/package.scala @@ -18,10 +18,14 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ query, + renderedTypeOf, + BooleanValue, + ComparisonRule, Expr, GenericIdentifier, Identifier, LiteralParam, + NumericValue, PainlessContext, PainlessOperandForm, PainlessScript, @@ -31,14 +35,19 @@ import app.softnetwork.elastic.sql.{ } import app.softnetwork.elastic.sql.`type`.{ SQLAny, + SQLBigInt, SQLBool, + SQLInt, SQLNumeric, + SQLSmallInt, SQLTemporal, + SQLTinyInt, SQLType, SQLTypeUtils, SQLTypes, SQLVarchar } +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.schema.Column import app.softnetwork.elastic.sql.query.{ @@ -282,6 +291,15 @@ package object cond { private val simpleOperand = """(?:[A-Za-z_][A-Za-z0-9_]*|-?\d+(?:\.\d+)?[lLfFdD]?|"(?:[^"\\]|\\.)*")""".r + /** A whole-number Painless literal (`GREATEST` / `LEAST` compare two of them when rendering). */ + private val WholeLiteral = """-?\d+[lL]?""".r + + /** How a simple `CASE WHEN ` compares (`Case.whenComparison`). */ + private[cond] sealed trait WhenComparison + private[cond] case object WhenAsResult extends WhenComparison + private[cond] case object WhenInstants extends WhenComparison + private[cond] case object WhenNumbers extends WhenComparison + case class NullIf(expr1: PainlessScript, expr2: PainlessScript) extends ConditionalFunction[SQLAny] { override def conditionalOp: ConditionalOp = NullIf @@ -346,7 +364,59 @@ package object cond { case other => other.out } - override def argTypes: List[SQLType] = args.map(argTypeOf) + /** What a string LITERAL argument is read as beside the other argument -- the ONE comparison + * rule (`ComparisonRule.readingOf`, the lead's ruling of 2026-10-05): `NULLIF(n, '3')` + * compares `n` with the number 3, `NULLIF(COALESCE(d, d2), '2024-01-31')` with a date. `None` + * for anything else, and for a literal that spells no value of the other's kind. + * + * Computed ONCE per `NULLIF` and memoised (its type, its rendering and the comparison it sits + * in all ask it; `ComparisonRule` says why with no lock); attaching the schema copies the + * `NULLIF` (`update`), so a resolved statement's reading is read against the resolved types. + */ + private[cond] def secondReading: Option[ComparisonRule.Reading] = { + var reading = _secondReading + if (reading eq null) { + reading = ComparisonRule.readingOf(expr2, ComparisonRule.declaredTypeOf(expr1)) + _secondReading = reading + } + reading + } + + private[cond] def firstReading: Option[ComparisonRule.Reading] = { + var reading = _firstReading + if (reading eq null) { + reading = ComparisonRule.readingOf(expr1, ComparisonRule.declaredTypeOf(expr2)) + _firstReading = reading + } + reading + } + + private[this] var _secondReading: Option[ComparisonRule.Reading] = null + private[this] var _firstReading: Option[ComparisonRule.Reading] = null + + /** A string literal argument READ as a date or a timestamp is rendered from that reading + * (`ComparisonRule.temporalPainless`), never through the per-document `String -> temporal` + * coercion the generic rendering binds for a literal: that coercion parses the literal's OWN + * text by the operand's type, so a date-time literal beside a DATE and a date-only one beside + * a TIMESTAMP -- both of which the reading reads, as PostgreSQL does -- threw on every + * document, and a computed column stored NULL (`ignore_failure`). + */ + override protected def ownArgumentPainless( + index: Int, + context: Option[PainlessContext] + ): Option[String] = + (index match { + case 0 => firstReading + case 1 => secondReading + case _ => None + }).collect { case t: ComparisonRule.TemporalReading => ComparisonRule.temporalPainless(t) } + + /** A literal argument counts as what it is read as: `NULLIF(n, '3')` compares two numbers. */ + override def argTypes: List[SQLType] = + List( + firstReading.map(_.sqlType).getOrElse(argTypeOf(expr1)), + secondReading.map(_.sqlType).getOrElse(argTypeOf(expr2)) + ) /** πŸ”΄ `NULLIF` REPORTS THE TYPE OF ITS FIRST ARGUMENT, not the super type of both. * @@ -369,9 +439,14 @@ package object cond { * Two questions, two derivations: `out` says how to compare, `reportedType` says what comes * out. A pair that cannot be compared at all is refused by [[typeMismatchError]] before either * matters. + * + * A first argument that is a string literal READ beside the second (`firstReading`) is the + * value it is read as, and is rendered as one: `NULLIF('2024-01-31', ts)` answers that + * midnight, a TIMESTAMP, as PostgreSQL types it -- not the VARCHAR the literal is written as, + * which a `CREATE TABLE … AS SELECT` declared, storing text. */ override def reportedType: SQLType = - reportedArgTypes.headOption.getOrElse(baseType) + firstReading.map(_.sqlType).getOrElse(reportedArgTypes.headOption.getOrElse(baseType)) /** The rejection this `NULLIF` earns when its two arguments RENDER types SQL does not compare * (issue #382, the LEAD RULING of 2026-09-23). @@ -461,7 +536,11 @@ package object cond { def typeMismatchError: Option[String] = { val left = argTypeOf(expr1) val right = argTypeOf(expr2) - if (temporalAgainstText(left, right)) + // A literal the other argument READS (`secondReading`) is a value of that argument's kind: + // comparable by construction, and rendered as that value. Asked first, by the query seam + // and the DDL one alike, so a computed column and a query read `NULLIF(n, '3')` the same. + if (secondReading.isDefined || firstReading.isDefined) None + else if (temporalAgainstText(left, right)) // The resolver goes first so a MALFORMED literal is named as such (`'x'` is not a date for // this field's format) rather than reported as a bare type mismatch; a WELL-FORMED one // falls through to a message that says what is actually wrong -- the engine, not the SQL. @@ -617,8 +696,17 @@ package object cond { ): String = { callArgs match { case List(rendered0, rendered1) => - val arg0 = operand(rendered0, context) - val arg1 = operand(rendered1, context) + // A string literal argument is the value the other argument READS it as -- the number + // or the boolean it spells, rendered as one (`secondReading` / `firstReading`). + def scalar(reading: Option[ComparisonRule.Reading], rendered: String): String = + reading.flatMap(ComparisonRule.scalarPainless).getOrElse(rendered) + val arg0 = operand(scalar(firstReading, rendered0), context) + // A second argument READ as a temporal is compared as the instant it denotes + // (`comparedMillis`) and is never the answer, so its value is not bound: it would be + // evaluated on every document for nothing. + val arg1 = + if (secondReading.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) rendered1 + else operand(scalar(secondReading, rendered1), context) // πŸ”΄ The SECOND argument is guarded too (issue #373, found by review) -- the same hole // `checkCase` had, one method up in this file. `NULLIF(CAST(d AS TIME), CAST(ts AS // TIME))` called the comparison with a null `arg1` for any document missing `ts`: @@ -630,35 +718,77 @@ package object cond { // answer is `a`. Short-circuiting `arg1 == null` to the NULL branch would invert that. def nullIf(comparison: String): String = s"$arg0 == null || ($arg1 != null && $comparison) ? null : $arg0" - val expr = - out match { - // πŸ”΄ `String.compareTo` takes a `String` and NOTHING else, so this arm is right only - // when BOTH operands render text -- and `leastCommonSuperType` answers VARCHAR for a - // MIXED pair too (`[BOOLEAN, KEYWORD]`, `[TIMESTAMP, KEYWORD]`). Without the guard - // `NULLIF(b, CAST(n AS KEYWORD))` and `NULLIF(d, CAST(i AS VARCHAR))` emitted - // `param1.compareTo(param2)` over a `Boolean` / `ZonedDateTime` receiver and threw - // PER DOCUMENT on ES 8.18.3 -- which in a computed column `ignoreFailure` SWALLOWS, - // so the column silently disappears. MEASURED: 32 corpus rows, all of them shapes - // that answered a value before the type fix of this same issue moved the supertype - // from the column's type to the chain's. They fall to the `==` arm below, which is - // what they emitted before and which Painless applies to any pair without throwing. - // - // The TEXT-against-NUMBER half of the same population is refused instead, by - // [[typeMismatchError]] -- `==` there would be a silent always-false. - case SQLTypes.Varchar if argTypes.forall(_.isText) => - nullIf(s"$arg0.compareTo($arg1) == 0") - // πŸ”΄ Keyed on what the RECEIVER's chain RENDERS, not on `out` (issue #373). - // `LocalTime` has no `isEqual`, and MEASURED on ES 8.18 before this, - // `NULLIF(CAST(d AS TIME), CAST('00:00:00' AS TIME))` failed the shard in a real - // `script_fields`. Neither `out` nor `expr1.out` says so: both are TIMESTAMP, the - // COLUMN's type -- `chainType` is the derivation that describes the rendering - // (#367), and it answers TIME. Two wrong keys were tried before measuring it. - case _: SQLTemporal if receiverIsTime => - nullIf(s"$arg0.compareTo($arg1) == 0") - case _: SQLTemporal => nullIf(s"$arg0.isEqual($arg1)") - // `==` is null-safe in Painless, so a numeric / boolean pair needs no guard. - case _ => s"$arg0 == $arg1 ? null : $arg0" + // Two temporals that do not render the same java.time type -- a DATE column's + // `ZonedDateTime` against a cast's `LocalDate` -- or a string literal READ as a temporal + // are compared as the instants they denote, a DATE as the instant its UTC day starts at + // (`ArithmeticExpression.comparedMillis`). `isEqual` failed between the first two + // (`NULLIF(COALESCE(d, d2), CAST('2024-01-31' AS DATE))`, all shards failed on 8.18.3); + // a literal is NEVER NULL, and ` != null` does not compile, so it is not + // guarded. + val temporalLiteral = Seq(firstReading, secondReading).exists { + case Some(_: ComparisonRule.TemporalReading) => true + case _ => false + } + def dateCarrying(t: SQLType): Boolean = + t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp + // (An ingest processor reads both arguments by their runtime class already -- the + // `processorInstant` arm below -- and its source is persisted: it keeps its bytes.) + val mixedTemporals = + !context.exists(_.isProcessor) && !receiverIsTime && + dateCarrying(ComparisonRule.declaredTypeOf(expr1)) && + dateCarrying(ComparisonRule.declaredTypeOf(expr2)) && { + val (java1, java2) = (argTypeOf(expr1), argTypeOf(expr2)) + !java1.isUnknown && !java2.isUnknown && java1 != java2 } + lazy val instants = + s"${ArithmeticExpression.comparedMillis(expr1, firstReading, arg0, context)} == " + + s"${ArithmeticExpression.comparedMillis(expr2, secondReading, arg1, context)}" + val expr = + if (secondReading.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) + s"$arg0 == null || ($instants) ? null : $arg0" + else if (temporalLiteral) + s"$arg0 == null || ($arg1 != null && $instants) ? null : $arg0" + else if (mixedTemporals) nullIf(instants) + else + out match { + // πŸ”΄ `String.compareTo` takes a `String` and NOTHING else, so this arm is right only + // when BOTH operands render text -- and `leastCommonSuperType` answers VARCHAR for a + // MIXED pair too (`[BOOLEAN, KEYWORD]`, `[TIMESTAMP, KEYWORD]`). Without the guard + // `NULLIF(b, CAST(n AS KEYWORD))` and `NULLIF(d, CAST(i AS VARCHAR))` emitted + // `param1.compareTo(param2)` over a `Boolean` / `ZonedDateTime` receiver and threw + // PER DOCUMENT on ES 8.18.3 -- which in a computed column `ignoreFailure` SWALLOWS, + // so the column silently disappears. MEASURED: 32 corpus rows, all of them shapes + // that answered a value before the type fix of this same issue moved the supertype + // from the column's type to the chain's. They fall to the `==` arm below, which is + // what they emitted before and which Painless applies to any pair without throwing. + // + // The TEXT-against-NUMBER half of the same population is refused instead, by + // [[typeMismatchError]] -- `==` there would be a silent always-false. + case SQLTypes.Varchar if argTypes.forall(_.isText) => + nullIf(s"$arg0.compareTo($arg1) == 0") + // πŸ”΄ Keyed on what the RECEIVER's chain RENDERS, not on `out` (issue #373). + // `LocalTime` has no `isEqual`, and MEASURED on ES 8.18 before this, + // `NULLIF(CAST(d AS TIME), CAST('00:00:00' AS TIME))` failed the shard in a real + // `script_fields`. Neither `out` nor `expr1.out` says so: both are TIMESTAMP, the + // COLUMN's type -- `chainType` is the derivation that describes the rendering + // (#367), and it answers TIME. Two wrong keys were tried before measuring it. + case _: SQLTemporal if receiverIsTime => + nullIf(s"$arg0.compareTo($arg1) == 0") + // πŸ”΄ In an INGEST processor a column argument is the incoming document's raw JSON -- + // a string or epoch milliseconds -- so `isEqual` failed on every document (`String` + // has none) and `ignore_failure` stored NULL: `NULLIF(d, d2) - d2` was NULL on every + // row (found by review, measured on 8.18.3). Both are compared as the instants they + // denote, read by their runtime class (`SQLTypeUtils.processorInstant`); NULLIF still + // answers its FIRST argument as it came. + case _: SQLTemporal if context.exists(_.isProcessor) => + nullIf( + s"${SQLTypeUtils.processorInstant(arg0)}" + + s".isEqual(${SQLTypeUtils.processorInstant(arg1)})" + ) + case _: SQLTemporal => nullIf(s"$arg0.isEqual($arg1)") + // `==` is null-safe in Painless, so a numeric / boolean pair needs no guard. + case _ => s"$arg0 == $arg1 ? null : $arg0" + } context match { case Some(ctx) => ctx.addParam(LiteralParam(expr)) match { @@ -676,10 +806,29 @@ package object cond { for { _ <- expr1.validate() _ <- expr2.validate() - _ <- Validator.validateTypesMatching(expr1.out, expr2.out) } yield () } + /** The TYPE rules of `NULLIF`, asked once the column types are known (the lead's ruling of + * 2026-10-05): [[typeMismatchError]] (issue #382), then the ONE comparison rule + * (`ComparisonRule`) -- `NULLIF(a, b)` compares `a = b`, so it is judged as that comparison: + * on the types the two arguments are DECLARED with, a DATE comparable with a TIMESTAMP. + * `NULLIF(COALESCE(d, d2), CAST('2024-01-31' AS DATE))` compares two dates, and is accepted + * like `NULLIF(d, CAST('2025-01-01' AS DATE))` always was (the lead's ruling of 2026-09-23). + */ + override def typeError: Option[String] = + typeMismatchError.orElse { + // judged on the readings this NULLIF already holds (`firstReading`, `secondReading`) + ComparisonRule + .mismatch( + ComparisonRule.comparedType(firstReading, expr1), + ComparisonRule.comparedType(secondReading, expr2) + ) + .map { case (left, right) => + s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" + } + } + override def update(request: query.SingleSearch): NullIf = { this.copy( expr1 = expr1 match { @@ -716,7 +865,25 @@ package object cond { s"$exprPart $whenThen$elsePart $END" } - override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(argTypes) + /** The type of the CASE's VALUE: the least common supertype of its `THEN` results and its + * `ELSE` -- never the `CASE ` operand, which is only compared ([[comparisonType]]), as + * [[reportedType]] already reads it. With the operand in, `CASE s WHEN 'a' THEN 1 ELSE 0 END` + * over a KEYWORD `s` answered the strings `"1"` / `"0"` and `CASE x WHEN 1.5 THEN 1 ELSE 0 + * END` over a DOUBLE `x` the doubles `1.0` / `0.0` while reporting BIGINT; PostgreSQL answers + * the integers 1 and 0. + */ + override def baseType: SQLType = + SQLTypeUtils.leastCommonSuperType( + conditions.map { case (_, res) => res.out } ++ default.map(_.out) + ) + + /** The type a simple CASE compares its operand and each `WHEN` value in: the common supertype + * of the operand and the results -- what [[baseType]] answered while it included the operand + * -- or the type a cast over the CASE sets, so every comparison renders exactly as it did. + * Only the VALUE's type moved ([[baseType]]). + */ + private[this] def comparisonType: SQLType = + if (out != baseType) out else SQLTypeUtils.leastCommonSuperType(argTypes) override def validate(): Either[String, Unit] = { // Story 22.2 (AD-6) β€” a CASE-WHEN condition is parsed by `case_condition`, which shares @@ -735,19 +902,54 @@ package object cond { "Filter in WHERE, or compute the flag in a separate query." ) else if (conditions.isEmpty) Left("CASE WHEN requires at least one condition") - else if ( - expression.isEmpty && conditions.exists { case (cond, _) => cond.out != SQLTypes.Boolean } - ) - Left("CASE WHEN conditions must be of type BOOLEAN") - else if ( - expression.isDefined && conditions.exists { case (cond, _) => - !SQLTypeUtils.matches(cond.out, expression.get.out) - } - ) - Left("CASE WHEN conditions must be of the same type as the expression") else Right(()) } + /** The TYPE rules of the conditions, asked once the column types are known (the lead's ruling + * of 2026-10-05) and on what each operand RENDERS (`renderedTypeOf`): asked when the statement + * was parsed, `CASE WHEN flag THEN …` was refused because a BOOLEAN column is `Any` there. A + * type that is still unknown is accepted: Elasticsearch answers. + */ + override def typeError: Option[String] = + expression match { + case None if conditions.exists { case (cond, _) => + val t = renderedTypeOf(cond) + t != SQLTypes.Boolean && !t.isUnknown + } => + Some("CASE WHEN conditions must be of type BOOLEAN") + // Each value is compared with the expression by the ONE comparison rule (`ComparisonRule`): + // on their DECLARED types, a DATE comparable with a TIMESTAMP, a string literal read by + // what it is compared with -- `CASE n WHEN '3'`, `CASE d WHEN CAST('2024-01-31' AS DATE)` + // -- judged on the readings this CASE already holds (`whenReadings`). + case Some(e) if { + val (declared, readings) = whenReadings(e) + val literalExpression = ComparisonRule.stringLiteral(e).isDefined + conditions.zip(readings).exists { case ((cond, _), reading) => + val left = + if (!literalExpression) declared + else + ComparisonRule + .readingOf(e, ComparisonRule.declaredTypeOf(cond)) + .map(_.sqlType) + .getOrElse(declared) + ComparisonRule + .mismatch(left, ComparisonRule.comparedType(reading, cond)) + .isDefined + } + } => + Some("CASE WHEN conditions must be of the same type as the expression") + case _ => None + } + + /** The least common supertype of the BRANCHES' reported types -- the THEN results and the ELSE, + * never the `CASE ` operand -- as `COALESCE` reports its arms': `CASE … THEN d ELSE d2 + * END` is a DATE when both are. + */ + override def reportedType: SQLType = + SQLTypeUtils.leastCommonSuperType( + conditions.map { case (_, result) => result.reportedType } ++ default.map(_.reportedType) + ) + /** Whether the CASE EXPRESSION -- the receiver of every `WHEN` comparison -- renders a * `java.time.LocalTime`. `out` reports the CASE's result type and cannot say (issue #373). */ @@ -773,12 +975,13 @@ package object cond { e: String, c: String, v: String, - candidateNullable: Boolean + candidateNullable: Boolean, + compared: SQLType ): String = { val guard = if (candidateNullable) s"$e != null && $c != null" else s"$e != null" // πŸ”΄ A TIME is decided by what the RECEIVER RENDERS, not by `out` (issue #373, found by - // review). `out` is the CASE's RESULT type -- the least common supertype of the expression, - // the conditions and the results -- and it reports TIMESTAMP for + // review). `out` -- then the least common supertype of the expression and the results, now + // [[comparisonType]] -- reports TIMESTAMP for // `CASE CAST(d AS TIME) WHEN CAST(ts AS TIME) …` whose operands are both `LocalTime`, so an // arm keyed on it was DEAD and that shape still emitted `isEqual`. MEASURED on ES 8.18: // `dynamic method [java.time.LocalTime, isEqual/1] not found`, the very failure this PR's @@ -786,7 +989,7 @@ package object cond { // describes the rendering -- the same key `NULLIF` needed. if (receiverIsTime) s"$guard && $e.compareTo($c) == 0 ? $v" else - out match { + compared match { case SQLTypes.Varchar => s"$guard && $e.compareTo($c) == 0 ? $v" case _: SQLTemporal => @@ -813,18 +1016,195 @@ package object cond { // wrong as written. if (default.isEmpty) s"(def)($rendered)" else rendered + /** How `CASE WHEN …` compares its expression with each value -- the ONE comparison + * rule (`ComparisonRule`) decides what is compared; this decides how it renders. + * + * The historical rendering converts BOTH sides to [[comparisonType]] -- the common type of the + * expression and the results, what `out` was until the CASE's value was typed by its results + * alone -- and is kept wherever that compares what SQL compares ([[WhenAsResult]]). It does + * not when: + * - the expression is a DATE or a TIMESTAMP and a value is a string literal read as one, the + * two render different java.time types, or the result is not a temporal: a `LocalDate` + * value against a column's `ZonedDateTime` failed the shard, and a numeric result compared + * `toEpochMilli()` with a `LocalDate` (`CASE d WHEN CAST('2024-01-31' AS DATE) THEN 1 ELSE + * 0 END` answered 0 for 2024-01-31) -- they are compared as the instants they denote + * ([[WhenInstants]]); + * - the expression is a number and the result is not: two numbers were compared as their + * strings (`"1.0"` against `"1"`) -- they are compared as numbers ([[WhenNumbers]]). An + * ingest processor keeps its (persisted) bytes unless a value is a string literal it now + * reads. + */ + private[this] def whenComparison( + expr: PainlessScript, + context: Option[PainlessContext] + ): WhenComparison = { + def dateCarrying(t: SQLType): Boolean = + t == SQLTypes.Date || t == SQLTypes.DateTime || t == SQLTypes.Timestamp + val (declared, readings) = whenReadings(expr) + val processor = context.exists(_.isProcessor) + // the type the two sides are converted to ([[WhenAsResult]]): the old `out` + val compared = comparisonType + if (receiverIsTime) WhenAsResult + else if (dateCarrying(declared)) { + val temporalReading = + readings.exists(_.exists(_.isInstanceOf[ComparisonRule.TemporalReading])) + // a KNOWN mismatch only: with no schema attached a type is unknown, and the statement + // keeps the rendering it always had + val mismatch = + (!compared.isInstanceOf[SQLTemporal] && !compared.isUnknown) || + conditions.exists { case (cond, _) => + val rendered = javaTypeOf(cond) + !rendered.isUnknown && rendered != javaTypeOf(expr) + } + if (temporalReading || (!processor && mismatch)) WhenInstants else WhenAsResult + } else if (declared.isNumber && !compared.isNumber && !compared.isUnknown) { + if (!processor || readings.exists(_.isDefined)) WhenNumbers else WhenAsResult + } else WhenAsResult + } + + /** The CASE expression's DECLARED type and what each `WHEN` value is read as beside it + * (`ComparisonRule.readingOf`), computed ONCE per CASE and memoised (`ComparisonRule` says why + * with no lock): its type rule, the comparison it is rendered as and every `WHEN` arm ask it, + * and a reading is a parse. Attaching the schema copies the CASE (`update`), so a resolved + * statement's readings are read against the resolved types. + */ + private[this] def expressionReadings: (SQLType, List[Option[ComparisonRule.Reading]]) = { + var readings = _expressionReadings + if (readings eq null) { + readings = expression match { + case Some(e) => readingsBeside(e) + case None => (SQLTypes.Any, conditions.map(_ => None)) + } + _expressionReadings = readings + } + readings + } + + private[this] var _expressionReadings: (SQLType, List[Option[ComparisonRule.Reading]]) = null + + private[this] def readingsBeside( + expr: PainlessScript + ): (SQLType, List[Option[ComparisonRule.Reading]]) = { + val declared = ComparisonRule.declaredTypeOf(expr) + (declared, conditions.map { case (cond, _) => ComparisonRule.readingOf(cond, declared) }) + } + + /** [[expressionReadings]] when `expr` IS the CASE expression -- the only one the renderings + * pass -- and computed for `expr` otherwise. + */ + private[this] def whenReadings( + expr: PainlessScript + ): (SQLType, List[Option[ComparisonRule.Reading]]) = + if (expression.exists(_ eq expr)) expressionReadings else readingsBeside(expr) + + /** The boolean CONSTANT a `WHEN` value is, if it is one: a string literal read as a boolean + * (`CASE b WHEN '1'`, `ComparisonRule.BooleanReading`), or the literal `TRUE` / `FALSE`. + */ + private[this] def booleanConstantOf( + value: PainlessScript, + reading: Option[ComparisonRule.Reading] + ): Option[Boolean] = + reading match { + case Some(ComparisonRule.BooleanReading(b)) => Some(b) + case Some(_) => None + case None => + value match { + case b: BooleanValue => Some(b.value) + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (b: BooleanValue) :: Nil => Some(b.value) + case _ => None + } + case _ => None + } + } + + /** The JAVA type an operand renders (#384's `Identifier.renderedType`): a DATE column's doc + * value is a `ZonedDateTime`, a cast to DATE a `LocalDate`. + */ + private[this] def javaTypeOf(operand: PainlessScript): SQLType = operand match { + case i: Identifier => i.renderedType + case other => other.out + } + + /** A `THEN` result, converted to the CASE's type -- the one rendering both comparisons share. + */ + private[this] def whenResult( + cond: PainlessScript, + res: PainlessScript, + context: Option[PainlessContext] + ): String = { + val name = + cond match { + case e: Expression => e.identifier.name + case f: FunctionWithIdentifier => f.identifier.name + case i: Identifier => i.name + case _ => "" + } + boxWhenNoDefault(res match { + case i: Identifier if i.name == name && name.nonEmpty => + i.withNullable(false) + SQLTypeUtils.coerce(i, out, context) + case _ => + SQLTypeUtils.coerce(res, out, context) + }) + } + + /** The `WHEN` arms of a [[WhenInstants]] or [[WhenNumbers]] comparison. The expression is + * rendered ONCE (a rendering is not idempotent) and, for instants, converted once; every value + * is guarded like the historical candidate (`checkCase`), and a NULL on either side matches + * nothing -- SQL's `NULL = NULL` is unknown, where Painless's `null == null` is true. + */ + private[this] def comparedWhens( + expr: PainlessScript, + kind: WhenComparison, + ctx: PainlessContext, + context: Option[PainlessContext] + ): String = { + val (_, readings) = whenReadings(expr) + def bind(rendered: String): String = ctx.addParam(LiteralParam(rendered)).getOrElse(rendered) + def instant(operand: PainlessScript, ref: String): String = + bind( + s"($ref != null ? (def)(${ArithmeticExpression.comparedMillis(operand, None, ref, context)}) : null)" + ) + val e = bind(expr.painless(context)) + val key = if (kind == WhenInstants) instant(expr, e) else e + conditions + .zip(readings) + .map { case ((cond, res), reading) => + val r = whenResult(cond, res, context) + val test = reading match { + case Some(t: ComparisonRule.TemporalReading) => s"$key == ${t.epochMillis}L" + case Some(scalar) => + s"$key == ${ComparisonRule.scalarPainless(scalar).getOrElse(cond.painless(context))}" + case None => + val c = bind(cond.painless(context)) + val value = if (kind == WhenInstants) instant(cond, c) else c + s"$value != null && $key == $value" + } + s"$key != null && $test ? $r" + } + .mkString(" : ") + } + override def painless(context: Option[PainlessContext] = None): String = { context match { case Some(ctx) => var cases = expression match { + case Some(expr) if whenComparison(expr, context) != WhenAsResult => + comparedWhens(expr, whenComparison(expr, context), ctx, context) case Some(expr) => // case with expression to evaluate - val e = SQLTypeUtils.coerce(expr, out, context) + // the operand and each value are converted to the type they are COMPARED in; the + // results, below, to the CASE's own (`out`) + val compared = comparisonType + val e = SQLTypeUtils.coerce(expr, compared, context) val expParam = ctx.addParam( LiteralParam(e) ) conditions - .map { case (cond, res) => + .zip(whenReadings(expr)._2) + .map { case ((cond, res), whenReading) => val name = cond match { case e: Expression => @@ -835,7 +1215,30 @@ package object cond { i.name case _ => "" } - val c = SQLTypeUtils.coerce(cond, out, context) + // A string literal is the value the expression READS it as -- the number or + // the boolean it spells (`ComparisonRule.readingOf`), brought to the type the + // two are compared in like any other value. `CASE b WHEN 'true'` used to parse + // `"true"` as a long (`Long.parseLong`) and fail the shard. + // + // A boolean CONSTANT -- such a reading, or `TRUE` / `FALSE` -- is that number + // itself (`1L` / `0L`): converted like a value, `(true ? 1L : 0L)` is a + // conditional over a constant, which Elasticsearch 6.8's Painless refuses to + // compile ("Extraneous conditional statement"). + val c = + booleanConstantOf(cond, whenReading) + .flatMap(SQLTypeUtils.booleanConstant(_, compared)) + .getOrElse( + whenReading + .flatMap(reading => + ComparisonRule + .scalarPainless(reading) + .map( + SQLTypeUtils + .coerce(_, reading.sqlType, compared, nullable = false, context) + ) + ) + .getOrElse(SQLTypeUtils.coerce(cond, compared, context)) + ) val r = boxWhenNoDefault(res match { case i: Identifier if i.name == name && name.nonEmpty => @@ -865,11 +1268,12 @@ package object cond { // (`addParam` returns the existing one), and `c != null` on a candidate // that cannot be null is redundant bytes, never a different answer. ctx.addParam(LiteralParam(c)) match { - case Some(bound) => checkCase(e, bound, r, candidateNullable = true) - case _ => checkCase(e, c, r, candidateNullable = false) + case Some(bound) => + checkCase(e, bound, r, candidateNullable = true, compared) + case _ => checkCase(e, c, r, candidateNullable = false, compared) } case _ => - checkCase(e, c, r, candidateNullable = false) + checkCase(e, c, r, candidateNullable = false, compared) } } .mkString(" : ") @@ -901,8 +1305,14 @@ package object cond { }) if (!cond.isInstanceOf[CriteriaWithConditionalFunction[_]] && cond.nullable) { ctx.addParam(LiteralParam(c)) match { - case Some(c) => s"$c ? $r" - case _ => s"$c ? $r" + // A VALUE condition -- a boolean column, a function of one -- is NULL + // where the document has no value, and a NULL condition is not true: the + // branch does not apply (SQL), where `param1 ? 1 : 0` threw on that + // document (a query failed; a computed column stored NULL). `==` is + // null-safe in Painless. A predicate already renders NULL as false. + case Some(c) if cond.isInstanceOf[Identifier] => s"$c == true ? $r" + case Some(c) => s"$c ? $r" + case _ => s"$c ? $r" } } else { s"$c ? $r" @@ -968,14 +1378,23 @@ package object cond { } } - /** N-ary numeric reducer shared by `GREATEST` / `LEAST`. + /** N-ary reducer shared by `GREATEST` / `LEAST`, over numbers or over dates and timestamps. * * Emits a right-folded nested ternary so a NULL argument is skipped and the whole expression * yields NULL only when every argument is NULL: pairwise(x, y) = (x == null ? y : (y == null ? x * : Math.{max|min}(x, y))) * - * Output is `SQLNumeric` because `Math.max` / `Math.min` only operate on numeric primitives. The - * base type narrows to the widest input numeric. + * Over numbers the type is the common type of ALL the arguments ([[numericType]]), as PostgreSQL + * types it, and every row answers it: per document the value is converted to it on every path + * ([[typedResult]]). Over dates and timestamps (the lead's ruling of 2026-10-05: every SQL + * engine compares them, and they failed in Elasticsearch before) the same fold runs over the + * epoch milliseconds each argument denotes -- a DATE its UTC day's start, through the conversion + * date arithmetic uses (`ArithmeticExpression.instantMillis`) -- so dates compare as instants + * whatever each one renders as (a doc value, a `LocalDate`, the clock), and a NULL is skipped + * exactly as it is for numbers. The result is the instant that wins, a UTC `ZonedDateTime` + * (TIMESTAMP is what it renders); per group, where a date is the number its metric is, it is + * that number. Its reported type is its arguments' common one: DATE for dates, TIMESTAMP when + * they mix. */ sealed trait NumericReducer extends TransformFunction[SQLAny, SQLNumeric] @@ -983,6 +1402,10 @@ package object cond { def values: List[PainlessScript] def operator: ConditionalOp protected def mathFn: String // "Math.max" | "Math.min" + protected def greatest: Boolean // GREATEST (the larger wins) or LEAST + + /** The comparison under which the left argument wins. */ + private def keepsLeft: String = if (greatest) ">=" else "<=" override def fun: Option[ConditionalOp] = Some(operator) @@ -994,37 +1417,106 @@ package object cond { override def inputType: SQLAny = SQLTypes.Any - override def baseType: SQLType = SQLTypeUtils.leastCommonSuperType(argTypes) match { - case n: SQLNumeric => n - case _ => outputType + /** Are the arguments dates and timestamps? Decided on what each one RENDERS (`renderedTypeOf`, + * the types the type rule reads), once the column types are known: until then a column is + * `Any`, and the reducer is the numeric one it always was. + */ + lazy val overDates: Boolean = values.exists(v => NumericReducer.dateCarrying(renderedTypeOf(v))) + + /** The numeric type of `GREATEST` / `LEAST`: the common type of ALL its arguments, as + * PostgreSQL resolves it (its CASE / UNION rule), which every row then has. + * - whole numbers only: the widest of them -- an INT beside a BIGINT is a BIGINT; + * - any other number -- a DOUBLE, a REAL, a NUMERIC of no known kind (`SQRT`, `ROUND`) -- + * makes it fractional: a DOUBLE, or a REAL when every fractional argument is a REAL. + * + * Each argument is read by the type it is DECLARED with (`ComparisonRule.declaredTypeOf`: its + * reported type, the metric for an aggregate), never by `out` ([[argTypes]]): a column + * resolved against its schema answers its COLUMN's type there whatever function it wears, so + * `CAST(n AS DOUBLE)` over an INT `n` read INT, the arguments looked whole, and the winner was + * chosen as a whole number -- the literal `1` itself where `n` is below 1 or NULL. A bare NULL + * has no type and is left out. + * + * `None` when an argument is not a number (dates take [[overDates]]), or when one's type is + * still unknown (no schema attached) and no fractional argument decides the type: an unknown + * column may hold fractions, so it is never read as a whole number. The reducer then renders + * as it always did. + */ + private[cond] lazy val numericType: Option[SQLNumeric] = { + val declared = values.map(ComparisonRule.declaredTypeOf).filterNot(_ == SQLTypes.Null) + val fractional = declared.filter(t => t.isNumber && !NumericReducer.wholeNumber(t)) + if (declared.isEmpty || declared.exists(t => !t.isNumber && !t.isUnknown)) None + else if (declared.forall(NumericReducer.wholeNumber)) + SQLTypeUtils.leastCommonSuperType(declared) match { + case whole: SQLNumeric => Some(whole) + case _ => None + } + else if (fractional.isEmpty) None + else if (fractional.forall(_ == SQLTypes.Real) && !declared.exists(_.isUnknown)) + Some(SQLTypes.Real) + else Some(SQLTypes.Double) } + override def baseType: SQLType = + if (overDates) SQLTypes.Timestamp + else + numericType.getOrElse(SQLTypeUtils.leastCommonSuperType(argTypes) match { + case n: SQLNumeric => n + case _ => outputType + }) + override def sql: String = s"$operator(${values.map(_.sql).mkString(", ")})" + /** An aggregate among the arguments makes this a per-group calculation, as it makes an + * arithmetic over them one (`BinaryFunction.hasAggregation`): `GREATEST(MAX(d), MAX(d2))` + * beside a GROUP BY is computed per group, where it used to be refused as a non-aggregated + * field while `GREATEST(MAX(d), MAX(d2)) - MIN(d)` was accepted. + */ + override def hasAggregation: Boolean = values.exists(_.hasAggregation) + override def checkIfNullable: Boolean = false override def validate(): Either[String, Unit] = - if (values.isEmpty) Left(s"$operator requires at least one argument") - else - // Accept numeric args and still-unresolved args (SQLAny / NULL, which - // extends SQLAny): a bare field has no known type until it is resolved - // against the index mapping, so we must not reject it here. Only reject - // args whose type is *definitively* non-numeric (string, temporal, - // boolean, …) β€” those would emit Math.{max,min}() on a non-numeric and - // fail at ES runtime. - values.find { v => - v.out match { - case _: SQLNumeric => false - case _: SQLAny => false - case _ => true - } - } match { - case Some(nonNumeric) => - Left( - s"$operator requires numeric arguments but got ${nonNumeric.out} for ${nonNumeric.sql}" - ) - case None => Right(()) + if (values.isEmpty) Left(s"$operator requires at least one argument") else Right(()) + + /** Numbers, or dates and timestamps -- one kind, as every SQL engine requires -- asked once the + * column types are known (the lead's ruling of 2026-10-05) and on what each argument RENDERS + * (`renderedTypeOf`). An argument whose type is still unknown (SQLAny / NULL, which extends + * SQLAny) is accepted: Elasticsearch answers. An argument of any other kind (a string, a + * boolean, a TIME), or numbers mixed with dates, is refused by name. + */ + override def typeError: Option[String] = { + val known = values + .map(v => (v: PainlessScript) -> renderedTypeOf(v)) + .filterNot(_._2.isInstanceOf[SQLAny]) + val numbers = known.filter(_._2.isInstanceOf[SQLNumeric]) + val dates = known.filter(a => NumericReducer.dateCarrying(a._2)) + val offenders = + known.find(a => + !a._2.isInstanceOf[SQLNumeric] && !NumericReducer.dateCarrying(a._2) + ) match { + case Some(other) => Seq(other) + case None if numbers.nonEmpty && dates.nonEmpty => Seq(dates.head, numbers.head) + case None => Nil } + // each argument is NAMED by the type DESCRIBE shows (a DATE column RENDERS as a TIMESTAMP) + def named(arg: PainlessScript): String = (arg match { + case aggregate: Identifier if aggregate.isAggregation => renderedTypeOf(aggregate) + case other => other.reportedType + }).typeId + if (offenders.isEmpty) None + else + Some( + s"$operator requires numeric arguments, or date and timestamp arguments, but got " + + offenders.map { case (arg, _) => s"${named(arg)} for ${arg.sql}" }.mkString(" and ") + ) + } + + /** Over numbers, [[numericType]] -- the type every row answers, so DESCRIBE, a `CREATE TABLE … + * AS SELECT` and JDBC metadata declare what is stored; otherwise the least common supertype of + * the arguments' REPORTED types, as `COALESCE` reports. + */ + override def reportedType: SQLType = + numericType.getOrElse(SQLTypeUtils.leastCommonSuperType(reportedArgTypes)) override def nullable: Boolean = values.forall(_.nullable) @@ -1049,40 +1541,204 @@ package object cond { case _ => false })) + // Pair each rendered arg with its nullability so non-nullable args (e.g. + // numeric literals, which render inline as primitives) are NOT guarded + // with `== null` β€” Painless rejects ` == null` at compile time. + // Right-fold pairwise: pairwise(a, pairwise(b, pairwise(c, …))). + // The combined sub-tree is itself nullable only when BOTH sides can be + // null, which lets us drop the guard on parent levels too. + private def fold(args: List[(String, Boolean)], integral: Boolean): (String, Boolean) = + args match { + case Nil => + throw new IllegalArgumentException(s"$operator requires at least one argument") + case (s, n) :: Nil => (s.trim, n) + case (s, xNullable) :: rest => + val x = s.trim + val (y, yNullable) = fold(rest, integral) + val best = if (integral) winner(x, y) else s"$mathFn($x, $y)" + val expr = + (xNullable, yNullable) match { + case (true, true) => s"($x == null ? $y : ($y == null ? $x : $best))" + case (true, false) => s"($x == null ? $y : $best)" + case (false, true) => s"($y == null ? $x : $best)" + case (false, false) => best + } + // result is null only if every branch can be null + (expr, xNullable && yNullable) + } + + /** The argument that wins, chosen by a comparison: Painless's `Math.max` / `Math.min` take + * `double`s only, so over whole numbers they answered `3.0` for `GREATEST(n, 2)` over an + * INTEGER `n` -- the right number in the wrong type (issue #380) -- and lost the precision of + * a BIGINT past 2^53. The winner is then converted with every other path ([[typedResult]]). + * Two whole-number literals are compared now, so the winner is the literal itself -- never a + * conditional over constants, which Elasticsearch 6.8 refuses as a whole script. + */ + private def winner(x: String, y: String): String = { + def literal(v: String): Option[BigInt] = + if (WholeLiteral.pattern.matcher(v).matches()) + scala.util.Try(BigInt(v.stripSuffix("L").stripSuffix("l"))).toOption + else None + (literal(x), literal(y)) match { + case (Some(a), Some(b)) => if (if (greatest) a >= b else a <= b) x else y + case _ => s"($x $keepsLeft $y ? $x : $y)" + } + } + + /** Are the arguments whole numbers -- [[numericType]] is one -- each rendered as a name, a + * literal or one parenthesised group, compared per document? Then the winner is chosen by + * [[winner]]. Per group -- no context, or a view's per-group calculation -- every metric is + * the double Elasticsearch computed, and the reducer keeps `Math.max` / `Math.min`; so does an + * argument rendered as a bare conditional (a function's NULL guard), which a comparison would + * read wrongly, and a bare `NULL`, which has no type and no comparison. + */ + private def overWholeNumbers( + callArgs: List[String], + context: Option[PainlessContext] + ): Boolean = + context.exists(!_.isTransform) && numericType.exists(NumericReducer.wholeNumber) && + !values.exists(ArithmeticExpression.bareNull) && + callArgs.forall(a => NumericReducer.oneValue(a.trim)) + + /** The value converted to [[numericType]], per document: ONE conversion every path goes through + * -- the winner, each NULL-skipping branch, a nested `GREATEST` / `LEAST`, a literal argument. + * + * The fold hands back the argument that survives as it is (`x == null ? y : …` answers `y` + * itself), and the winner as it is, so a column answered two classes: `GREATEST(n, x)` gave + * the INT `n` where the DOUBLE `x` is NULL (`Integer` -2 beside `Double` values), and + * `GREATEST(GREATEST(n, 0), x)` the `Integer` 0. MEASURED on Elasticsearch 8.18.3 and 6.8.23. + * `Math.max` hid it while it answered a double, except in the NULL-skipping branches. + * + * Per group -- no context, or a view's per-group calculation -- every metric is the double + * Elasticsearch computed, and the rendering is the one it always was. With a number literal + * among the arguments the value is never NULL and is converted as it stands; otherwise it is + * read once, under a name, behind a NULL guard whose NULL branch is that name -- a `def`, as + * the converted value is, where a `null` literal would make the conditional unusable as the + * argument of an outer `Math.max` (`Cannot cast null to a primitive type [double]`). + */ + private def typedResult(value: String, context: Option[PainlessContext]): String = + perDocumentType(context) match { + case Some((ctx, t)) => + val cast = SQLTypeUtils.painlessType(t) + val v = value.trim + if (values.exists(NumericReducer.numberLiteral)) + s"(($cast) ${if (NumericReducer.oneValue(v)) v else s"($v)"})" + else { + val isName = + ArithmeticExpression.isBareName(v) && (v.head.isLetter || v.head == '_') + val name = if (isName) v else ctx.bindLocal(v, "lv") + s"($name == null ? $name : (def) (($cast) $name))" + } + case _ => value + } + + /** The context a value is converted in, and the type it is converted to: per document only. */ + private def perDocumentType( + context: Option[PainlessContext] + ): Option[(PainlessContext, SQLNumeric)] = + for { + ctx <- context.filterNot(_.isTransform) + t <- numericType + } yield (ctx, t) + + /** A literal argument as a literal of [[numericType]] where its own spelling is not one: a + * whole number past the range of an `int` is a `long` literal (`3000000000L`) beside a BIGINT, + * which Painless refuses to compile without its suffix (`Invalid int constant`). + */ + private def typedArgument( + rendered: String, + argument: PainlessScript, + context: Option[PainlessContext] + ): String = + perDocumentType(context) match { + case Some((_, SQLTypes.BigInt)) if NumericReducer.numberLiteral(argument) => + val r = rendered.trim + if ( + WholeLiteral.pattern.matcher(r).matches() && !r.endsWith("L") && !r.endsWith("l") && + !BigInt(r).isValidInt + ) s"${r}L" + else rendered + case _ => rendered + } + override def toPainlessCall( callArgs: List[String], context: Option[PainlessContext] - ): String = { - // Pair each rendered arg with its nullability so non-nullable args (e.g. - // numeric literals, which render inline as primitives) are NOT guarded - // with `== null` β€” Painless rejects ` == null` at compile time. - // Right-fold pairwise: pairwise(a, pairwise(b, pairwise(c, …))). - // The combined sub-tree is itself nullable only when BOTH sides can be - // null, which lets us drop the guard on parent levels too. - def fold(args: List[(String, Boolean)]): (String, Boolean) = - args match { - case Nil => - throw new IllegalArgumentException(s"$operator requires at least one argument") - case (s, n) :: Nil => (s.trim, n) - case (s, xNullable) :: rest => - val x = s.trim - val (y, yNullable) = fold(rest) - val expr = - (xNullable, yNullable) match { - case (true, true) => s"($x == null ? $y : ($y == null ? $x : $mathFn($x, $y)))" - case (true, false) => s"($x == null ? $y : $mathFn($x, $y))" - case (false, true) => s"($y == null ? $x : $mathFn($x, $y))" - case (false, false) => s"$mathFn($x, $y)" - } - // result is null only if every branch can be null - (expr, xNullable && yNullable) - } + ): String = + if (overDates) overDatesPainless(callArgs, context) + else { + val typedArgs = + callArgs.zip(values).map { case (rendered, argument) => + typedArgument(rendered, argument, context) + } + typedResult( + typedArgs match { + case Nil => + throw new IllegalArgumentException(s"$operator requires at least one argument") + case x :: Nil => x + case _ => + fold( + typedArgs.zip(values.map(nullableArgument(_, context))), + overWholeNumbers(typedArgs, context) + )._1 + }, + context + ) + } - callArgs match { - case Nil => - throw new IllegalArgumentException(s"$operator requires at least one argument") - case x :: Nil => x - case _ => fold(callArgs.zip(values.map(nullableArgument(_, context))))._1 + /** The same fold over the epoch milliseconds each date denotes, then the instant that wins. + * + * Per document each argument is converted once, under a name, behind its own NULL guard (a + * NULL is skipped by the fold, never converted), and the winner is handed back as the UTC + * `ZonedDateTime` it denotes. Per group -- no context, or a view's per-group calculation -- a + * metric IS the epoch milliseconds Elasticsearch computed for its date, and may be NULL in a + * group filter: it is read through the same conversion (a DATE metric is its UTC day's start, + * as date arithmetic reads it per group: a `date` field accepts a date-time, and its MAX keeps + * that time), behind a NULL guard where it can be NULL, and the winner stays that number, as a + * date aggregate is there. A NULL literal can never win: it is left out. + */ + private def overDatesPainless( + callArgs: List[String], + context: Option[PainlessContext] + ): String = { + if (callArgs.isEmpty) + throw new IllegalArgumentException(s"$operator requires at least one argument") + val perDocument = context.filterNot(_.isTransform) + val millis = callArgs.zip(values).filterNot(a => ArithmeticExpression.nullLiteral(a._2)).map { + case (ref, argument) => + val nullable = nullableArgument(argument, context) + perDocument match { + case Some(ctx) => + val name = + if (ArithmeticExpression.isBareName(ref.trim)) ref.trim + else ctx.bindLocal(ref, "lv") + val converted = ArithmeticExpression.instantMillis(argument, name, context) + val guarded = + if (nullable) s"($name == null ? null : (def)($converted))" else converted + (ctx.bindLocal(guarded, "lv"), nullable) + case None if argument.isAggregation => + val metric = ref.trim + val converted = ArithmeticExpression.instantMillis(argument, metric, context) + // A NULL metric (a group filter's, over a group with none of its values) is handed + // back as it is -- the same NULL, which the fold skips. The group filter's gate + // (`MetricSelectorScript.disqualifyRendering`) refuses a rendering that spells + // `? null :` or reads `def`, so neither is written: the conditional is a `def` by its + // first branch, the metric itself. + if (converted == metric) (metric, nullable) + else if (nullable) (s"($metric == null ? $metric : $converted)", nullable) + else (converted, nullable) + case None => + (ArithmeticExpression.instantMillis(argument, ref.trim, context), nullable) + } + } + if (millis.isEmpty) return if (perDocument.isEmpty) "((def) null)" else "null" + val (winner, nullable) = fold(millis, integral = false) + perDocument match { + case Some(ctx) => + val name = if (nullable) ctx.bindLocal(winner, "lv") else s"($winner)" + val instant = s"Instant.ofEpochMilli((long) $name).atZone(ZoneId.of('Z'))" + if (nullable) s"($name == null ? null : $instant)" else instant + case None => winner } } } @@ -1090,6 +1746,7 @@ package object cond { case class Greatest(values: List[PainlessScript]) extends NumericReducer { override def operator: ConditionalOp = Greatest override protected def mathFn: String = "Math.max" + override protected def greatest: Boolean = true override def update(request: query.SingleSearch): Greatest = this.copy(values = values.map { @@ -1101,6 +1758,7 @@ package object cond { case class Least(values: List[PainlessScript]) extends NumericReducer { override def operator: ConditionalOp = Least override protected def mathFn: String = "Math.min" + override protected def greatest: Boolean = false override def update(request: query.SingleSearch): Least = this.copy(values = values.map { @@ -1108,4 +1766,68 @@ package object cond { case other => other }) } + + object NumericReducer { + + /** A type that carries a date -- DATE, TIMESTAMP, DATETIME, or a temporal function's TEMPORAL + * -- which `GREATEST` / `LEAST` compare as instants. A TIME carries none. + */ + private[sql] def dateCarrying(t: SQLType): Boolean = + t == SQLTypes.Date || t == SQLTypes.Timestamp || t == SQLTypes.DateTime || + t == SQLTypes.Temporal + + /** A whole-number type: TINYINT, SMALLINT, INT or BIGINT. */ + private[cond] def wholeNumber(t: SQLType): Boolean = t match { + case _: SQLTinyInt | _: SQLSmallInt | _: SQLInt | _: SQLBigInt => true + case _ => false + } + + /** A number literal -- bare, or the one function of the nameless identifier the parser wraps an + * operand in -- which is never NULL. + */ + private[cond] def numberLiteral(v: PainlessScript): Boolean = v match { + case _: NumericValue[_] => true + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (_: NumericValue[_]) :: Nil => true + case _ => false + } + case _ => false + } + + /** A rendering that is ONE value wherever it is placed: a name, a literal, or a single + * parenthesised group. + */ + private[cond] def oneValue(rendered: String): Boolean = + simpleOperand.pattern.matcher(rendered).matches() || (rendered.startsWith("(") && { + var depth = 0 + var i = 0 + var closedEarly = false + while (i < rendered.length && !closedEarly) { + rendered.charAt(i) match { + case '(' => depth += 1 + case ')' => + depth -= 1 + if (depth == 0 && i < rendered.length - 1) closedEarly = true + case _ => + } + i += 1 + } + !closedEarly && depth == 0 + }) + + /** Is this operand -- or the nameless identifier carrying it -- `GREATEST` / `LEAST` over + * dates? Per group its value is the epoch milliseconds of the date that wins + * (`ArithmeticExpression.epochMillis`). + */ + private[sql] def overDates(operand: PainlessScript): Boolean = operand match { + case reducer: NumericReducer => reducer.overDates + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (reducer: NumericReducer) :: Nil => reducer.overDates + case _ => false + } + case _ => false + } + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala index 6690afcd0..e46139854 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/convert/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ query, + renderedTypeOf, Alias, DateMathRounding, Expr, @@ -28,6 +29,8 @@ import app.softnetwork.elastic.sql.{ Updateable } import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} +import app.softnetwork.elastic.sql.function.cond.functionsOf +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression package object convert { @@ -112,7 +115,7 @@ package object convert { } case _ => // do nothing } - val ret = SQLTypeUtils.coerce(base, value.baseType, targetType, value.nullable, context) + val ret = SQLTypeUtils.coerce(base, sourceType, targetType, sourceNullable, context) val bloc = ret.startsWith("{") && ret.endsWith("}") val retWithBrackets = if (bloc) ret else s"{ $ret }" if (!safe) ret @@ -151,11 +154,90 @@ package object convert { } } + /** The arithmetic this conversion converts, when its operand IS one: `(n + x)::BIGINT`, whose + * operand is the nameless identifier carrying that one arithmetic. + */ + private def arithmetic: Option[ArithmeticExpression] = value match { + case a: ArithmeticExpression => Some(a) + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (a: ArithmeticExpression) :: Nil => Some(a) + case _ => None + } + case _ => None + } + + /** The type the conversion converts FROM: what an arithmetic operand COMPUTES (its own type, + * which a cast no longer overwrites -- `ArithmeticExpression.cast`), or the operand's type. + * + * πŸ”΄ The nameless identifier around an arithmetic folds its chain from the arithmetic's INPUT + * type, so it answered NUMERIC for `(n + x)`, and `NUMERIC -> BIGINT` has no conversion arm: + * the cast was silently ignored (`(n + x)::BIGINT` answered 3.25 on main). + */ + private def sourceType: SQLType = arithmetic.map(_.out).getOrElse(value.baseType) + + /** Can the value converted be NULL? Asked of the chain's innermost PRODUCER: the nameless + * identifier around an arithmetic answers `false` whatever its operands hold, so the + * conversion was emitted unguarded -- `((long) )` failed with `Cannot cast null to a + * primitive type`, `String.valueOf()` answered the text "null", and ` != 0` + * answered TRUE. Guarded, the value is bound once and the conversion applies to it when it is + * not NULL (`SQLTypeUtils.coerce`'s nullable form): `(v == null ? null : conversion)`. + */ + private def sourceNullable: Boolean = + value.nullable || (value match { + case i: Identifier if i.name.trim.isEmpty => + i.functions.lastOption.exists { + case a: ArithmeticExpression => a.nullable + case _ => false + } + case a: ArithmeticExpression => a.nullable + case _ => false + }) + override def roundingScript: Option[String] = DateMathRounding(targetType) override def dateMathScript: Boolean = isTemporal + + /** A NUMBER converted to a DATE is refused by name, as PostgreSQL refuses it (`cannot cast type + * integer to date`) -- `(d - d2)::DATE`, `CAST(n AS DATE)`, `CAST(YEAR(d) AS DATE)`. A number + * is no date, and no conversion makes it one: the number used to pass through unconverted + * under a DATE label (`(d - d2)::DATE` answered -1, 2 and 0). + * + * Decided on what the operand RENDERS ([[convertedType]]): `YEAR(d)` is a number although its + * column is a date. Asked once the column types are known, like every type rule; an operand + * whose type is still unknown is accepted. A number cast to a TIMESTAMP keeps core's + * epoch-millisecond conversion. + */ + override def typeError: Option[String] = + if (targetType != SQLTypes.Date) None + else { + val source = convertedType + if (!source.isNumber) None + else + Some( + s"Type mismatch: cannot cast ${source.typeId} to DATE in expression: $sql; a number " + + "is not a date: add a number of days to a date instead" + ) + } + + /** The type of the value converted, as it renders: its outermost function's own type -- never + * that function's `out`, which this conversion's constructor sets to the TARGET type + * (`value.cast(targetType)`), so that `CAST(1 AS DATE)` and `CAST(YEAR(d) AS DATE)` would read + * as dates -- an aggregate's metric, or the column's type. + */ + private def convertedType: SQLType = value match { + case i: Identifier if !i.isAggregation && i.functions.nonEmpty => i.functions.head.baseType + case other => renderedTypeOf(other) + } } + /** Every conversion type error in an expression ([[Conversion.typeError]]), wherever a conversion + * sits in it -- the walk a computed column's resolved table asks, as it asks the date-arithmetic + * rules (`ArithmeticExpression.typeErrorsOf`). + */ + def typeErrorsOf(chain: FunctionChain): Seq[String] = + functionsOf(chain).collect { case c: Conversion => c }.flatMap(_.typeError).distinct + case object Cast extends Expr("CAST") with TokenRegex case object TryCast extends Expr("TRY_CAST") with TokenRegex { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala index 14567aa12..713a8b3d3 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/math/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql.function import app.softnetwork.elastic.sql.{ query, + renderedTypeOf, Expr, IntValue, PainlessContext, @@ -26,7 +27,15 @@ import app.softnetwork.elastic.sql.{ TokenRegex, Updateable } -import app.softnetwork.elastic.sql.`type`.{SQLNumeric, SQLType, SQLTypes} +import app.softnetwork.elastic.sql.`type`.{ + SQLBigInt, + SQLInt, + SQLNumeric, + SQLSmallInt, + SQLTinyInt, + SQLType, + SQLTypes +} package object math { @@ -99,6 +108,43 @@ package object math { arg: PainlessScript ) extends MathematicalFunction { override def args: List[PainlessScript] = List(arg) + + /** `ABS` is typed by its argument, as PostgreSQL's `abs(integer) -> integer`: `ABS(n)` over an + * INT column is an INT, so a literal beside it is read as that type's value -- `ABS(n) = + * '1.5'` is refused by name like `n = '1.5'`, where NUMERIC read `'1.5'` as a number -- and a + * `CREATE TABLE … AS SELECT` declares an INT, where NUMERIC could not create the index. An + * argument whose type is not a known number keeps NUMERIC. + * + * The type REPORTED, not the one Painless is handed (`out`, still NUMERIC): what a consumer + * converts this value to is unchanged. + */ + override def reportedType: SQLType = mathOp match { + case Abs => + arg.reportedType match { + case n: SQLNumeric => n + case _ => super.reportedType + } + case _ => super.reportedType + } + + /** `ABS` of a whole number answers a whole number of its argument's type per document, as its + * type says -- Painless's `Math.abs` takes a `double` only, so it answered `3.0` for an INT + * `-3`. The argument is a parameter or a literal, read at most twice. Per group -- no context, + * or a view's per-group calculation -- every metric is the double Elasticsearch computed, and + * the rendering is the one it always was. + */ + override def toPainlessCall( + callArgs: List[String], + context: Option[PainlessContext] + ): String = + (mathOp, renderedTypeOf(arg), callArgs) match { + case (Abs, _: SQLTinyInt | _: SQLSmallInt | _: SQLInt | _: SQLBigInt, List(a)) + if context.exists(!_.isTransform) => + val x = s"(${a.trim})" + s"($x < 0 ? -$x : $x)" + case _ => super.toPainlessCall(callArgs, context) + } + override def update(request: query.SingleSearch): MathematicalFunctionWithOp = { arg match { case updatable: Updateable => diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala index d86d35295..cdc37d110 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/package.scala @@ -324,6 +324,20 @@ package object function { def checkIfNullable: Boolean = args.exists(_.nullable) || args.exists(argRendersNullOnFailure) + /** The Painless of the argument at `index` when this function renders it ITSELF, in place of + * the conversion [[painless]] gives an argument with no parameter of its own (a literal) -- + * which, with a context, BINDS that conversion to `argTypes(index)` as a parameter of the + * script, evaluated on every document. `None`, the default, leaves the argument to it. + * + * `NULLIF` renders this way a string literal it reads as a date or a timestamp: from its + * READING, never through a per-document parse of the literal's text (see `NullIf`). + */ + protected def ownArgumentPainless( + index: Int, + context: Option[PainlessContext] + ): Option[String] = + None + override def painless(context: Option[PainlessContext]): String = { context match { case Some(ctx) => @@ -412,8 +426,10 @@ package object function { }).getOrElse { if (a.nullable) s"arg$i" else - SQLTypeUtils - .coerce(a.painless(context), a.baseType, argTypes(i), nullable = false, context) + ownArgumentPainless(i, context).getOrElse( + SQLTypeUtils + .coerce(a.painless(context), a.baseType, argTypes(i), nullable = false, context) + ) } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala index 89b330323..bd1d58e6c 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/function/time/package.scala @@ -74,10 +74,10 @@ package object time { } } - override def validate(): Either[String, Unit] = interval.checkType(out) match { - case Left(err) => Left(err) - case Right(_) => Right(()) - } + /** The interval's unit against the type it applies to, asked once the column types are known + * (the lead's ruling of 2026-10-05), never when the statement is parsed. + */ + override def typeError: Option[String] = interval.checkType(out).left.toOption override def toPainless(base: String, idx: Int, context: Option[PainlessContext]): String = { context match { @@ -380,6 +380,20 @@ package object time { case _ => } super.toPainlessCall(callArgs, context) + // πŸ”΄ A DAY truncation of a date-only receiver is the receiver itself, and `truncatedTo` + // would fail on it: `DATE_TRUNC(ts, DAY) = '2024-01-31'` (or `> '…'`, or `= CAST(… AS + // DATE)`) narrows `ts` to a `LocalDate` to meet the date, and `LocalDate.truncatedTo` does + // not exist (all shards failed, MEASURED on 8.18.3). Rendered as the start of that day, as + // a date: the same value, through methods a `LocalDate` has -- and a `ZonedDateTime` + // receiver misjudged as date-only fails loudly on `atStartOfDay` rather than silently + // keeping its time. + // + // ⚠️ DAY only. HOUR, MINUTE and SECOND keep `truncatedTo`: on a receiver NARROWED to its + // date the time is already gone, and SQL compares `DATE_TRUNC(ts, HOUR)` with a date as + // the instant the date starts at, which a date cannot be compared as -- that comparison + // still fails loudly, rather than answering for every hour of the day. + case TimeUnit.DAYS if receiverIsDateOnly => + ".atStartOfDay(ZoneId.of('Z')).toLocalDate()" case _ => super.toPainlessCall(callArgs, context) } } @@ -610,12 +624,14 @@ package object time { * and `DATE_DIFF` read a calendar date through it too, off a row-level operand that is not a * column: a zoned value there may be Elasticsearch 6.8's doc value, which is no `java.time` * type until `withZoneSameInstant` (as in [[utcZoned]]) returns the `ZonedDateTime` it wraps. + * Date arithmetic (`ArithmeticExpression`) reads such an operand through it for the same + * reason: one conversion, shared. * * `ref` is read up to four times, so it is a name or an expression cast to `def` (the methods * below are resolved at runtime). NULL stays NULL: the MONTH, QUARTER and YEAR calls bind this * value in the script's prologue, which runs before the null guard. */ - private[time] def utcInstant(ref: String): String = + private[sql] def utcInstant(ref: String): String = s"($ref == null ? null : $ref instanceof LocalDate ? $ref.atStartOfDay(ZoneId.of('Z')) : $ref instanceof LocalDateTime ? $ref.atZone(ZoneId.of('Z')) : $ref.withZoneSameInstant(ZoneId.of('Z')))" /** A column a SEARCH script reads, as a genuine `java.time.ZonedDateTime` in UTC; NULL stays diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala index 6b904e023..7a9d55a35 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/operator/math/ArithmeticExpression.scala @@ -18,10 +18,15 @@ package app.softnetwork.elastic.sql.operator.math import app.softnetwork.elastic.sql._ import app.softnetwork.elastic.sql.`type`._ -import app.softnetwork.elastic.sql.function.{BinaryFunction, TransformFunction} +import app.softnetwork.elastic.sql.function.{BinaryFunction, FunctionChain, TransformFunction} +import app.softnetwork.elastic.sql.function.aggregate.AggregateFunction +import app.softnetwork.elastic.sql.function.cond.{functionsOf, Case, NullIf, NumericReducer} +import app.softnetwork.elastic.sql.function.time.DateDiff import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.query.NestedElement +import scala.util.Try + case class ArithmeticExpression( left: PainlessScript, operator: ArithmeticOperator, @@ -36,7 +41,24 @@ case class ArithmeticExpression( override def inputType: SQLNumeric = SQLTypes.Numeric override def outputType: SQLNumeric = SQLTypes.Numeric - override def applyType(in: SQLType): SQLType = in + /** Date arithmetic hands its OWN type to the chain it heads, so that a cast over it converts from + * what it computes: `(d - d2)::TIMESTAMP` is a number of days cast to TIMESTAMP, which is core's + * epoch-millisecond conversion, as for any other number. Every other arithmetic keeps handing on + * the type it is given, as it always has. + */ + override def applyType(in: SQLType): SQLType = dateArithmetic.flatMap(_.runtimeType).getOrElse(in) + + /** A cast OVER this expression converts what it computes; it does not change what it computes. + * + * πŸ”΄ A conversion's constructor casts its operand (`value.cast(targetType)`), and on an + * arithmetic that set `out` -- the type the arithmetic coerces its OWN operands to. So `(x * + * 2)::INT` truncated `x` BEFORE multiplying (1.6 * 2 answered 2, not 3), and `(n + x)::BIGINT` + * -- whose operands are nullable, so not coerced at all -- answered 3.25 while the conversion, + * reading the cast type back as its source, emitted nothing. MEASURED on main. The arithmetic + * now keeps its own type and the conversion applies core's cast rule to its RESULT + * (`convert.Conversion`), as SQL engines do: `(n + x)::BIGINT` is 3 for 1 + 2.25. + */ + override def cast(targetType: SQLType): SQLType = out override def sql: String = { val expr = s"${left.sql}$operator${right.sql}" @@ -70,10 +92,76 @@ case class ArithmeticExpression( * The non-`Identifier` arm keeps today's `_.out` exactly: a nested arithmetic renders what its * own `out` says, and a literal's `out` already carries any cast applied to it. */ - private def argTypeOf(operand: PainlessScript): SQLType = operand match { - case i: Identifier => i.chainType - case other => other.out - } + private def argTypeOf(operand: PainlessScript): SQLType = + literalNumber(operand) + .map(_.sqlType) + .getOrElse(operand match { + case i: Identifier => ArithmeticExpression.identifierType(i) + case other => other.out + }) + + /** A string LITERAL beside a NUMBER is the number it spells -- the ONE comparison rule's reading + * (`ComparisonRule.readingOf`), as PostgreSQL reads `n + '1'`: 4 for n = 3, where the + * concatenation `"31"` was answered. Beside a whole-number operand only a whole number is one: + * PostgreSQL refuses `n + '1.5'` over an integer `n` (`invalid input syntax for type integer`), + * and so does the operand match here, the literal staying text. Beside anything else it stays + * what it is (a temporal beside it is date arithmetic, which reads it itself). + * + * Each operand's reading is computed ONCE per expression and memoised (`ComparisonRule` says why + * with no lock): every `argTypes` (so every `out`) and every rendering asks it, and a reading is + * a parse. + */ + private def literalNumber(operand: PainlessScript): Option[ComparisonRule.NumberReading] = + if (operand eq left) { + var number = _leftNumber + if (number eq null) { + number = numberBeside(left, right) + _leftNumber = number + } + number + } else if (operand eq right) { + var number = _rightNumber + if (number eq null) { + number = numberBeside(right, left) + _rightNumber = number + } + number + } else numberBeside(operand, left) + + private[this] var _leftNumber: Option[ComparisonRule.NumberReading] = null + private[this] var _rightNumber: Option[ComparisonRule.NumberReading] = null + + private def numberBeside( + operand: PainlessScript, + other: PainlessScript + ): Option[ComparisonRule.NumberReading] = + ComparisonRule + .readingOf(operand, ComparisonRule.declaredTypeOf(other)) + .collect { case number: ComparisonRule.NumberReading => number } + + /** The temporal a string literal operand is read as beside the other one + * (`ArithmeticExpression.temporalString`: `d - '2024-01-31'`), computed ONCE per expression and + * memoised -- the date-arithmetic rendering asks it per operand and per guard. + */ + private def literalTemporal(operand: PainlessScript): Option[ComparisonRule.TemporalReading] = + if (operand eq left) { + var temporal = _leftTemporal + if (temporal eq null) { + temporal = ArithmeticExpression.temporalString(left, right) + _leftTemporal = temporal + } + temporal + } else if (operand eq right) { + var temporal = _rightTemporal + if (temporal eq null) { + temporal = ArithmeticExpression.temporalString(right, left) + _rightTemporal = temporal + } + temporal + } else ArithmeticExpression.temporalString(operand, left) + + private[this] var _leftTemporal: Option[ComparisonRule.TemporalReading] = null + private[this] var _rightTemporal: Option[ComparisonRule.TemporalReading] = null override def argTypes: List[SQLType] = args.map(argTypeOf) @@ -125,11 +213,73 @@ case class ArithmeticExpression( * Measured on a clean clone at `origin/main`, so it is pre-existing and UNFILED. The documented * way to truncate is to compute the quotient into a column and cast THAT column. */ - override def baseType: SQLType = { - val folded = SQLTypeUtils.leastCommonSuperType(argTypes) - if (operator == DIVIDE && (folded.isNumber || folded.isUnknown)) SQLTypes.Double else folded + override def baseType: SQLType = dateArithmetic.flatMap(_.runtimeType).getOrElse { + val types = argTypes + // πŸ”΄ `+` and `-` answer UNKNOWN while an operand's type is unknown -- a column before its + // schema is attached, a field no schema describes. They are the two operators a DATE may + // take, so the fold could not know the answer: `d + 7` folded `[ANY, BIGINT]` to BIGINT, and + // `d + 7 > CURRENT_DATE` was refused as BIGINT against DATE before the column's type was ever + // read. `*`, `/` and `%` keep their fold: no date survives them. + if ((operator == ADD || operator == SUBTRACT) && types.exists(_.isUnknown)) SQLTypes.Any + else { + val folded = SQLTypeUtils.leastCommonSuperType(types) + if (operator == DIVIDE && (folded.isNumber || folded.isUnknown)) SQLTypes.Double else folded + } } + /** What the date-arithmetic rules make of this expression (see [[ArithmeticExpression]] 's + * companion), or `None` when neither operand is a DATE, a TIMESTAMP or a DATETIME -- every such + * expression keeps the derivation and the rendering it always had. + * + * πŸ”΄ Decided on the operands' DECLARED types, never on `out`. Elasticsearch stores DATE, + * TIMESTAMP and DATETIME alike as a `date`, so the type a query hands Painless (`runtimeType`) + * is TIMESTAMP for all three, and the difference between a DATE and a TIMESTAMP -- days between + * two dates, or a fraction of a day between two instants -- lives only in the declaration. + * Folding `out` is what typed `MAX(d) - MIN(d)` TIMESTAMP and answered it in milliseconds. + * + * Computed once per instance: `out` reads `baseType` on every call, and the operands of an + * instance do not change -- resolution (`update`) builds a new one. + */ + private[sql] lazy val dateArithmetic: Option[ArithmeticExpression.DateArithmetic] = + ArithmeticExpression.dateArithmetic(this) + + /** The type a consumer DECLARING a column for this expression reports: DATE for a date moved by + * whole days, although Painless renders it as the UTC instant that day starts at. + */ + override def reportedType: SQLType = dateArithmetic.flatMap(_.reportedType).getOrElse(out) + + /** The by-name refusal the DATE-ARITHMETIC rules give this expression, if any: a temporal operand + * under `*`, `/` or `%`, two temporals added, a number minus a temporal, or a temporal combined + * with something that is neither a number nor a temporal. + */ + def dateArithmeticError: Option[String] = + dateArithmetic.collect { case ArithmeticExpression.Refused(message) => message } + + /** The by-name refusal this expression earns once the types of its operands are known, if any: + * [[dateArithmeticError]], or -- for any other arithmetic -- two operands whose types do not + * match, the rule `validate()` used to apply when the statement was parsed, where every column + * is still `Any`. + * + * πŸ”΄ Asked AFTER the schema is attached (`SingleSearch.validateResolved`), never at parse time: + * no type is refused before the column types are known (the lead's ruling of 2026-10-05). Each + * operand's type is the one it RENDERS ([[ArithmeticExpression.identifierType]]), not `out`, + * which a resolved column answers with its column's type whatever function it wears. + */ + override def typeError: Option[String] = + dateArithmetic match { + case Some(_) => dateArithmeticError + case None => + Validator.validateTypesMatching(argTypeOf(left), argTypeOf(right)).left.toOption + } + + /** [[dateArithmeticError]], here and in every arithmetic among the operands, however deep. */ + private[sql] def dateArithmeticErrors: Seq[String] = + dateArithmeticError.toSeq ++ args.flatMap { + case nested: ArithmeticExpression => nested.dateArithmeticErrors + case chain: FunctionChain => ArithmeticExpression.typeErrorsOf(chain) + case _ => Nil + } + /** Is this the numeric division the ruling above governs? Keyed on the coercion TARGET rather * than on [[baseType]] so that an explicit `CAST(a / b AS INTEGER)` -- which sets `out` -- is * integer division again, with no arm of its own. @@ -215,10 +365,28 @@ case class ArithmeticExpression( * leaves the whole estate green, so nothing here distinguishes the two for a non-`Identifier` * operand. Measured, recorded on #382, and not smuggled in. */ - private def renderedType(operand: PainlessScript): SQLType = operand match { - case i: Identifier => i.chainType - case other => other.baseType - } + private def renderedType(operand: PainlessScript): SQLType = + literalNumber(operand) + .map(_.sqlType) + .getOrElse(operand match { + case i: Identifier => ArithmeticExpression.identifierType(i) + case other => other.baseType + }) + + /** An operand as it renders, a string literal read as a number ([[literalNumber]]) as that + * number. + */ + private def operandPainless( + operand: PainlessScript, + idx: Int, + context: Option[PainlessContext] + ): String = + literalNumber(operand) + .map(_.painless) + .getOrElse(operand match { + case t: TransformFunction[_, _] => t.toPainless("", idx + 1, context) + case _ => operand.painless(context) + }) /** πŸ”΄ A numeric coercion is skipped only over a NULLABLE operand, and the reason is exactly what * Painless refuses -- a primitive cast over a null-guarded expression, in BOTH directions, @@ -247,7 +415,45 @@ case class ArithmeticExpression( * the operand is known non-null. */ private def coercionSkipped(operand: PainlessScript, target: SQLType): Boolean = - operand.nullable && renderedType(operand).isNumber && target.isNumber + mayBeNull(operand) && renderedType(operand).isNumber && target.isNumber + + /** A `CASE` or a `NULLIF` written as an operand -- the nameless identifier the parser wraps it in + * -- is ONE value, and its rendering is a CONDITIONAL (`c ? a : b`, `x == 1 ? null : x`) over + * which `+`, `-`, `*`, `/` and `%` bind tighter: spliced as it stands, `CASE WHEN n > 0 THEN n + * ELSE 0 END + 1` was `param2 ? n : 0 + 1`, the ELSE branch plus one -- 3, 5, 1, 1, 10 where + * PostgreSQL answers 4, 6, 1, 1, 11 (issue #380), SQL parentheses or not. + */ + private def conditionalOperand(operand: PainlessScript): Boolean = operand match { + case i: Identifier if i.name.trim.isEmpty && !i.isAggregation => + i.functions.headOption.exists { + case _: Case | _: NullIf => true + case _ => false + } + case _ => false + } + + /** Can this operand be NULL when it runs? Its own nullability, or -- for a `CASE` or a `NULLIF` + * operand, whose nameless identifier answers `false` whatever it holds -- the function's: a + * `NULLIF` exists to produce NULL, and a `CASE` is NULL when a branch can be or it has no + * `ELSE`. Unguarded, `NULLIF(n, 1) + 1` threw on the very row it nulls (issue #380). + */ + private def mayBeNull(operand: PainlessScript): Boolean = + operand.nullable || (operand match { + case i: Identifier if conditionalOperand(i) => + i.functions.head match { + case c: Case => c.nullable + case _ => true + } + case _ => false + }) + + /** A conditional operand's rendering, parenthesised to be the ONE value it is wherever an + * operator or a conversion is applied to it; anything else, and a bare name, as it stands. + */ + private def asOneValue(operand: PainlessScript, rendered: String): String = + if (conditionalOperand(operand) && !ArithmeticExpression.isBareName(rendered.trim)) + s"($rendered)" + else rendered private def isFloating(t: SQLType): Boolean = t == SQLTypes.Double || t == SQLTypes.Real @@ -298,15 +504,32 @@ case class ArithmeticExpression( private def needsDoubleCast(target: SQLType): Boolean = floatingDivision(target) && !args.exists(operand => isFloating(emittedType(operand, target))) - override def validate(): Either[String, Unit] = { + /** STRUCTURE only: the operands. Every TYPE rule of this expression -- the date-arithmetic rules + * and the operand match alike -- is [[typeError]], asked once the column types are known (the + * lead's ruling of 2026-10-05: no refusal before the column types are known). Asked here, at + * parse time, it refused `d + 7 > CURRENT_DATE` as BIGINT against DATE before `d` had a type. + */ + override def validate(): Either[String, Unit] = for { _ <- left.validate() _ <- right.validate() - _ <- Validator.validateTypesMatching(left.out, right.out) } yield () - } - override def nullable: Boolean = left.nullable || right.nullable + /** Is an operand a NULL literal, bare or cast, in date arithmetic? Then the expression is NULL on + * every row, whatever the other operand holds. + */ + private def nullValued: Boolean = + dateArithmetic.exists { + case ArithmeticExpression.NullOperand => true + case _: ArithmeticExpression.Computed => + ArithmeticExpression.nullLiteral(left) || ArithmeticExpression.nullLiteral(right) + case _ => false + } + + /** A NULL literal beside a temporal makes the expression NULL on every row, whatever the other + * operand holds, so a consumer must guard it. + */ + override def nullable: Boolean = mayBeNull(left) || mayBeNull(right) || nullValued override def toPainless(base: String, idx: Int, context: Option[PainlessContext]): String = { context match { @@ -315,6 +538,12 @@ case class ArithmeticExpression( ctx.addParam(right) case _ => } + if (nullValued) return s"$base${nullOperandPainless(context)}" + dateArithmetic match { + case Some(computed: ArithmeticExpression.Computed) => + return s"$base${dateArithmeticPainless(computed, idx, context)}" + case _ => + } if (nullable) { // The coercion target, asked ONCE -- see `floatingDivision` for why that matters. val target = out @@ -331,10 +560,8 @@ case class ArithmeticExpression( else SQLTypeUtils.coerce(rendered, renderedType(operand), target, nullable = false, context) - def render(operand: PainlessScript): String = operand match { - case t: TransformFunction[_, _] => coerced(t.toPainless("", idx + 1, context), operand) - case _ => coerced(operand.painless(context), operand) - } + def render(operand: PainlessScript): String = + coerced(asOneValue(operand, operandPainless(operand, idx, context)), operand) /** A rendering that IS a Painless name needs no local of its own. */ def isBareName(e: String): Boolean = @@ -397,7 +624,7 @@ case class ArithmeticExpression( } val leftParam = - if (left.nullable && !isBareName(l)) bind(l, s"lv$idx") else l + if (mayBeNull(left) && !isBareName(l)) bind(l, s"lv$idx") else l /** πŸ”΄ The divisor is bound when it is not a bare NAME, even if it is not nullable -- because * the zero guard reads it a SECOND time. `$rightParam` appears in `($rightParam == 0)` and @@ -410,7 +637,7 @@ case class ArithmeticExpression( * A non-nullable, non-name LEFT operand needs no such binding: it is read once. */ val rightParam = - if (!isBareName(r) && (right.nullable || (floatingDivision(out) && divisorMayBeZero))) + if (!isBareName(r) && (mayBeNull(right) || (floatingDivision(out) && divisorMayBeZero))) bind(r, s"rv$idx") else r @@ -427,8 +654,8 @@ case class ArithmeticExpression( */ val lhs = if (needsDoubleCast(target)) s"((double) $leftParam)" else leftParam val guards = - (if (left.nullable) List(s"$leftParam == null") else Nil) ::: - (if (right.nullable) List(s"$rightParam == null") else Nil) ::: + (if (mayBeNull(left)) List(s"$leftParam == null") else Nil) ::: + (if (mayBeNull(right)) List(s"$rightParam == null") else Nil) ::: (if (floatingDivision(target) && divisorMayBeZero && !divisorIsLiteralZero) List(s"$rightParam == 0") else Nil) @@ -444,10 +671,185 @@ case class ArithmeticExpression( s"$base${painless(context)}" } - override def painless(context: Option[PainlessContext]): String = { + override def painless(context: Option[PainlessContext]): String = + if (nullValued) nullOperandPainless(context) + else + dateArithmetic match { + case Some(computed: ArithmeticExpression.Computed) => + dateArithmeticPainless(computed, 0, context) + case _ => numericPainless(context) + } + + /** A NULL literal beside a temporal: NULL. A `bucket_script` returns a Number, and a bare `null` + * is an Object to Painless -- `Cannot cast from [java.lang.Object] to [java.lang.Number]` at + * compile time (MEASURED, ES 8.18.3); typed `def`, it compiles and the group has no value, as it + * has for any NULL operand. + */ + private def nullOperandPainless(context: Option[PainlessContext]): String = { + val value = if (context.isEmpty) "((def) null)" else "null" + if (group) s"($value)" else value + } + + /** Date arithmetic, rendered in the milliseconds Elasticsearch stores a `date` in -- ONE formula + * per rule for every venue, which differ only in how an operand becomes a number of milliseconds + * and how a temporal result is handed back. + * + * - Per group (`bucket_script`, `bucket_selector`, a view's per-group calculation), which is + * every rendering without a context, an aggregate is the metric Elasticsearch computed: a + * number, epoch millis for a date. A temporal result is handed back as epoch millis too, + * which is what a date aggregate is there and what a `bucket_selector` compares a date with. + * - Per document, a column is the `date` value Elasticsearch hands the script (`toInstant()` + * exists on the `JodaCompatibleZonedDateTime` of 6.8 as on the `ZonedDateTime` of 7 and up), + * or, in an ingest processor, the raw value of the document, parsed as a CAST parses it. + * Anything else is read by its RUNTIME type (`DateDiff.utcInstant`): `CURRENT_DATE` is a + * `LocalDate` in a query and the current instant in a processor. A temporal result is the + * UTC `ZonedDateTime` it denotes -- the one temporal an ingest processor may assign to a + * field. + * + * A DATE operand is its UTC calendar day: the days between two dates are the days between their + * days, and a date moved by days starts at the start of its day. A NULL operand makes the result + * NULL, behind the same guard every arithmetic emits. + */ + private def dateArithmeticPainless( + shape: ArithmeticExpression.Computed, + idx: Int, + context: Option[PainlessContext] + ): String = { + import ArithmeticExpression.{ + isBareName, + DayMillis, + DaysBetween, + DaysShift, + FractionalDayMillis, + FractionalDaysBetween + } + + var declarations = "" + + def grouped(e: String): String = if (group) s"($e)" else e + + // the operand as it renders -- no coercion: the conversion below starts from what it IS + // A string literal that is a number counts as that number (`d + '1'`), so it renders as one. + def rendered(operand: PainlessScript): String = + (operand match { + case i: Identifier if i.functions.isEmpty => context.flatMap(_.get(i)) + case _ => ArithmeticExpression.numericString(operand).map(_._1) + }).getOrElse { + val r = operand match { + case t: TransformFunction[_, _] => t.toPainless("", idx + 1, context) + case _ => operand.painless(context) + } + if (PainlessOperandForm.placeable(r)) r else context.flatMap(_.get(operand)).getOrElse(r) + } + + // An operand that can be NULL when it runs: one whose own nullability says so, or one that + // READS a document field through a function. The nameless identifier around a `CASE`, a + // `COALESCE` or a `NULLIF` answers `nullable = false` whatever its branches hold, so it went + // unguarded and a NULL branch failed the script on `.toInstant()` instead of answering NULL. + // (An AGGREGATE is the metric Elasticsearch computed: a group with no value never runs the + // per-group script -- the `bucket_script`'s gap policy skips it -- so it keeps its own answer.) + def mayBeNull(operand: PainlessScript): Boolean = + operand.nullable || (operand match { + case i: Identifier if !i.isAggregation => + i.dependencies.nonEmpty || i.functions.exists(_.nullable) + case _ => false + }) + + // A string literal read as the temporal it spells beside the other operand (`d - + // '2024-01-31'`): the instant it denotes, known here, so it is a constant -- never NULL, never + // parsed per document (and read once per expression, `literalTemporal`). + def literalInstant(operand: PainlessScript): Option[Long] = + literalTemporal(operand).map(_.epochMillis) + + // a nullable operand the guard reads is evaluated once, under a name + def named(operand: PainlessScript, fallback: String): String = + literalInstant(operand).map(millis => s"${millis}L").getOrElse { + val r = rendered(operand) + if (!mayBeNull(operand) || isBareName(r)) r + else + context match { + case Some(ctx) => ctx.bindLocal(r, "lv") + case None => + declarations += s"def $fallback = $r; " + fallback + } + } + + // Per group there is no document: every operand is a number already, and so is the result. + val perGroup = context.isEmpty + + // the UTC day of a temporal operand, and the instant it denotes -- a DATE its day's start -- + // in epoch milliseconds: the conversion GREATEST / LEAST over dates shares + def day(operand: PainlessScript, ref: String): String = + literalInstant(operand) + .map(millis => s"${Math.floorDiv(millis, 86400000L)}L") + .getOrElse(ArithmeticExpression.epochDay(operand, ref, context)) + + def instant(operand: PainlessScript, ref: String): String = + literalInstant(operand) + .map(millis => s"${millis}L") + .getOrElse(ArithmeticExpression.instantMillis(operand, ref, context)) + + def days(ref: String, fractional: Boolean): String = + if (fractional) s"Math.round($ref * $FractionalDayMillis)" else s"$ref * $DayMillis" + + // Per document, a whole number of days or of milliseconds is a `long` again; per group it stays + // the double a `bucket_script` returns anyway, and a `bucket_selector` compares as it is. + def integral(e: String): String = if (perGroup) e else s"((long) $e)" + + val l = named(left, s"lv$idx") + val r = named(right, s"rv$idx") + + val value = shape match { + case DaysBetween => integral(s"(${day(left, l)} - ${day(right, r)})") + case FractionalDaysBetween => + s"((${instant(left, l)} - ${instant(right, r)}) / $FractionalDayMillis)" + case DaysShift(temporalLeft, _, fractional) => + val ((temporal, t), n) = if (temporalLeft) ((left, l), r) else ((right, r), l) + val moved = + s"(${instant(temporal, t)} ${operator.painless(context)} ${days(n, fractional)})" + if (perGroup) moved else s"Instant.ofEpochMilli(${integral(moved)}).atZone(ZoneId.of('Z'))" + } + + // (a literal instant is a primitive constant: never NULL, and ` == null` does not compile) + val guards = + (if (mayBeNull(left) && literalInstant(left).isEmpty) List(s"$l == null") else Nil) ::: + (if (mayBeNull(right) && literalInstant(right).isEmpty) List(s"$r == null") else Nil) + val body = + if (guards.isEmpty) value else s"(${guards.mkString(" || ")}) ? null : (def)($value)" + s"$declarations${grouped(body)}" + } + + private def numericPainless(context: Option[PainlessContext]): String = { val target = out - val l = SQLTypeUtils.coerce(left, target, context) - val r = SQLTypeUtils.coerce(right, target, context) + // πŸ”΄ An aggregate is coerced FROM what it computes (`AggregateFunction.outputTypeOf`), never + // from the column it aggregates: per group it is the metric Elasticsearch already computed, a + // number, and `COUNT(d) - 1` used to read the DATE column's type and emit + // `params.count_d.toInstant().toEpochMilli() - 1`. + def coerced(operand: PainlessScript): String = operand match { + case metric: Identifier if metric.isAggregation => + SQLTypeUtils.coerce( + metric.painless(context), + ArithmeticExpression.identifierType(metric), + target, + nullable = false, + context + ) + case literal if literalNumber(literal).isDefined => + val number = literalNumber(literal).get + SQLTypeUtils.coerce(number.painless, number.sqlType, target, nullable = false, context) + case conditional if conditionalOperand(conditional) => + SQLTypeUtils.coerce( + asOneValue(conditional, conditional.painless(context)), + conditional.baseType, + target, + nullable = false, + context + ) + case _ => SQLTypeUtils.coerce(operand, target, context) + } + val l = coerced(left) + val r = coerced(right) val core = s"$l ${operator.painless(context)} $r" /* The non-nullable rendering: both operands are literals or system values, so each is coerced * to `out` and a DOUBLE `out` is all it takes to make `SELECT 10 / 3` answer 3.333 instead of @@ -500,3 +902,353 @@ case class ArithmeticExpression( None } } + +/** The date-arithmetic rules: what `+` and `-` mean when an operand is a DATE, a TIMESTAMP or a + * DATETIME, and which combinations are type errors. + * + * | Expression | Result | + * |:-------------------------------------------------|:-------------------------------------------| + * | DATE - DATE | the calendar days between the two (BIGINT) | + * | TIMESTAMP - TIMESTAMP, or a DATE and a TIMESTAMP | the milliseconds between, in days (DOUBLE) | + * | DATE +/- a whole number n | the date n days later / earlier (DATE) | + * | DATE +/- a fractional number n | n days later / earlier, as an instant | + * | TIMESTAMP +/- a number n | n days later / earlier (TIMESTAMP) | + * | n + a temporal | the temporal + n | + * | a NULL literal beside a temporal | NULL | + * | two temporals added; a temporal under * / or %; | a type error, refused by name before | + * | n - a temporal; a temporal and a non-number | anything runs | + * + * They are the lead's ruling of 2026-10-04, aligned on the SQL engines a JDBC client knows: + * PostgreSQL, DuckDB, Oracle and Snowflake subtract two dates into days; PostgreSQL, DuckDB, + * Oracle and BigQuery move a date by a number of days; Oracle, whose DATE carries a time of day, + * answers fractional days, and moves a date by a fractional number of days into a time of day; + * PostgreSQL, DuckDB, Oracle, Trino, Snowflake and SQL Server refuse the other combinations. + * + * A temporal operand is recognised by its DECLARED type (`Token.reportedType`): a TIME, and an + * operand whose type is unknown (a column with no schema attached), leave the expression to the + * arithmetic it always was -- except under `*`, `/` and `%`, which refuse a temporal operand + * whatever the other one is. + */ +object ArithmeticExpression { + + /** One day, in the milliseconds Elasticsearch stores a `date` in. */ + private val DayMillis = "86400000L" + + /** The same, as a floating-point number, for a fraction of a day. */ + private val FractionalDayMillis = "86400000.0" + + /** What the date-arithmetic rules make of an expression. */ + private[sql] sealed trait DateArithmetic { + + /** The type the rendering produces (`Token.baseType`): a temporal result is rendered as the + * instant it denotes, so it is a TIMESTAMP whether it is reported as a DATE or not -- the same + * collapse `SQLTypeUtils.runtimeType` applies to a DATE column. + */ + def runtimeType: Option[SQLType] = None + + /** The type a consumer declaring a column reports (`Token.reportedType`). */ + def reportedType: Option[SQLType] = runtimeType + } + + /** A value the date-arithmetic rendering computes -- as opposed to a NULL, or a refusal. */ + private[sql] sealed trait Computed extends DateArithmetic + + /** DATE - DATE: the calendar days between the UTC days of the two dates. */ + private[sql] case object DaysBetween extends Computed { + override def runtimeType: Option[SQLType] = Some(SQLTypes.BigInt) + } + + /** A difference with an instant on one side at least: the milliseconds between, in days. */ + private[sql] case object FractionalDaysBetween extends Computed { + override def runtimeType: Option[SQLType] = Some(SQLTypes.Double) + } + + /** A temporal moved by a number of days, the number on either side of a `+`. + * + * @param temporalLeft + * whether the temporal is the LEFT operand + * @param calendarDate + * whether the temporal is a DATE, which is moved from the start of its day + * @param fractional + * whether the number of days has a fractional type, whose fraction becomes a time of day + */ + private[sql] final case class DaysShift( + temporalLeft: Boolean, + calendarDate: Boolean, + fractional: Boolean + ) extends Computed { + override def runtimeType: Option[SQLType] = Some(SQLTypes.Timestamp) + override def reportedType: Option[SQLType] = + Some(if (calendarDate && !fractional) SQLTypes.Date else SQLTypes.Timestamp) + } + + /** A NULL literal beside a temporal: NULL, with the type the expression derives otherwise. */ + private[sql] case object NullOperand extends DateArithmetic + + /** A combination no rule allows. */ + private[sql] final case class Refused(message: String) extends DateArithmetic + + /** One operand, as the rules see it. */ + private sealed trait Operand + private case object CalendarDate extends Operand + private case object Moment extends Operand + private final case class Days(fractional: Boolean) extends Operand + private case object NullLiteral extends Operand + private case object Undecided extends Operand + private case object NonNumber extends Operand + + private def temporal(operand: Operand): Boolean = operand == CalendarDate || operand == Moment + + /** The type an IDENTIFIER operand of an arithmetic computes: an aggregate is the metric it + * produces ([[AggregateFunction.outputTypeOf]]: `COUNT(d)` counts, whatever `d` is), anything + * else is what its chain renders (#382's `chainType`). + * + * πŸ”΄ The ONE derivation the arithmetic asks, for its own type and for the coercion of each + * operand alike (#292): reading the aggregated COLUMN's type typed `COUNT(d) - 1` as a TIMESTAMP + * and emitted `params.count_d.toInstant()` on the count. + */ + private[sql] def identifierType(i: Identifier): SQLType = renderedTypeOf(i) + + /** The type an operand is DECLARED with -- its REPORTED type, an aggregate's being the metric it + * produces: the ONE derivation the comparison rule reads too (`ComparisonRule.declaredTypeOf`). + * + * An Elasticsearch `date` field elasticsql never declared is a TIMESTAMP at the declaration seam + * itself (`schema.IndexField`), so it keeps its time here as everywhere else, and every chain + * over it follows: `MAX(ts)`, `COALESCE(ts, ts2)`, `CASE … THEN ts …`. + */ + private def declaredType(operand: PainlessScript): SQLType = + ComparisonRule.declaredTypeOf(operand) + + /** `NULL`, bare or cast: its VALUE is NULL in every venue. */ + private[sql] def nullLiteral(operand: PainlessScript): Boolean = operand match { + case Null => true + case i: Identifier => i.name.trim.isEmpty && i.functions.lastOption.contains(Null) + case _ => false + } + + /** A BARE `NULL`, which has no type of its own. A cast gives one -- `CAST(NULL AS DATE)` is a + * DATE, `CAST(NULL AS INT)` a whole number -- and a typed NULL is then judged by its type, + * whatever its value. + */ + private[sql] def bareNull(operand: PainlessScript): Boolean = operand match { + case Null => true + case i: Identifier => i.name.trim.isEmpty && i.functions == List(Null) + case _ => false + } + + /** A string LITERAL whose text is a number, as the Painless literal of that number and whether it + * is fractional (the lead's ruling of 2026-10-05) -- the ONE reader the comparison rule uses too + * (`ComparisonRule.numberOf`). + * + * Beside a DATE or a TIMESTAMP it is a number of days -- elasticsql's OWN rule, not + * PostgreSQL's: `d + '1'` is the next day and `d - '1.5'` a day and a half earlier, where + * PostgreSQL 16 refuses `d + '1'` (`operator is not unique: date + unknown`), reads the `'1'` of + * `d - '1'` as a date (`invalid input syntax for type date`) and reads `ts + '1'` as one second. + * A whole number is a `long` literal (`'99999999999'` does not fit an `int`), a fractional one + * the `double` it denotes. + */ + private[math] def numericString(operand: PainlessScript): Option[(String, Boolean)] = + ComparisonRule + .stringLiteral(operand) + .flatMap(ComparisonRule.numberOf) + .map(number => number.painless -> number.fractional) + + /** A string LITERAL read as the temporal it spells beside the temporal `other` -- the comparison + * rule's reading (`ComparisonRule.readingOf`), so `d - '2024-01-31'` is DATE minus DATE, as + * PostgreSQL reads it, and `ts - '2024-01-31T10:00:00Z'` TIMESTAMP minus TIMESTAMP. Beside + * anything else, or spelling no date, the literal is no temporal. + */ + private[math] def temporalString( + operand: PainlessScript, + other: PainlessScript + ): Option[ComparisonRule.TemporalReading] = + ComparisonRule.stringLiteral(operand).flatMap { _ => + operandOf(other) match { + case CalendarDate | Moment => + ComparisonRule.readingOf(operand, declaredType(other)).collect { + case t: ComparisonRule.TemporalReading => t + } + case _ => None + } + } + + private def operandOf(o: PainlessScript): Operand = + if (bareNull(o)) NullLiteral + else + numericString(o) match { + case Some((_, fractional)) => Days(fractional) + case None => + declaredType(o) match { + case SQLTypes.Date => CalendarDate + case SQLTypes.Timestamp | SQLTypes.DateTime | SQLTypes.Temporal => Moment + case SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int | SQLTypes.BigInt => + Days(fractional = false) + case SQLTypes.Double | SQLTypes.Real | SQLTypes.Numeric => Days(fractional = true) + // a TIME is outside these rules, and an unknown type cannot be judged + case t if t == SQLTypes.Time || t.isUnknown => Undecided + case _ => NonNumber + } + } + + /** One operand beside the other: a string literal that spells a date or a timestamp beside a + * temporal is that temporal ([[temporalString]]); anything else is what it is alone. + */ + private def operandOf(o: PainlessScript, other: PainlessScript): Operand = + temporalString(o, other) match { + case Some(t) if t.sqlType == SQLTypes.Date => CalendarDate + case Some(_) => Moment + case None => operandOf(o) + } + + private[math] def isCalendarDate(o: PainlessScript): Boolean = operandOf(o) == CalendarDate + + private[sql] def isBareName(e: String): Boolean = + e.nonEmpty && e.forall(c => c.isLetterOrDigit || c == '_') + + /** A TEMPORAL operand as the epoch milliseconds it denotes, in the venue `context` renders for -- + * the ONE conversion date arithmetic and `GREATEST` / `LEAST` over dates share (#292): + * + * - per group (`bucket_script`, `bucket_selector`: no context) an aggregate is the metric + * Elasticsearch computed, a number -- in a view's per-group calculation too -- and so is a + * value this module renders from metrics there: a date moved by days, `GREATEST` / `LEAST` + * over dates; + * - per document, a column is the `date` value Elasticsearch hands the script (`toInstant()` + * exists on the `JodaCompatibleZonedDateTime` of 6.8 as on the `ZonedDateTime` of 7 and up), + * or, in an ingest processor, whatever the incoming document holds -- the raw JSON string or + * number of a column, or what a function made of it -- read by its RUNTIME class + * (`SQLTypeUtils.processorInstant`), never by the type it is declared with; + * - anything else is read by its RUNTIME type (`DateDiff.utcInstant`): `CURRENT_DATE` is a + * `LocalDate` in a query and the current instant in a processor. + * + * `ref` is the operand's rendering: a name whenever the operand can be NULL (the caller guards + * it). + */ + private[sql] def epochMillis( + operand: PainlessScript, + ref: String, + context: Option[PainlessContext] + ): String = { + val perGroup = context.isEmpty + val processor = context.exists(_.isProcessor) + operand match { + case metric: Identifier + if metric.isAggregation && (perGroup || context.exists(_.isTransform)) => + ref + case nested: ArithmeticExpression + if nested.dateArithmetic.exists(_.isInstanceOf[DaysShift]) => + if (perGroup) ref else s"$ref.toInstant().toEpochMilli()" + case reducer + if (perGroup || context.exists(_.isTransform)) && NumericReducer.overDates(reducer) => + ref + case column: Identifier if column.name.trim.nonEmpty && column.functions.isEmpty => + if (processor) s"${SQLTypeUtils.processorInstant(ref)}.toInstant().toEpochMilli()" + else s"$ref.toInstant().toEpochMilli()" + case other => + context match { + case None => + s"${SQLTypeUtils.coerce(ref, other.baseType, SQLTypes.Timestamp, nullable = false, None)}.toInstant().toEpochMilli()" + case Some(ctx) => + val name = + if (isBareName(ref)) ref + else ctx.addParam(LiteralParam(ref)).getOrElse(s"((def) ($ref))") + if (processor) s"${SQLTypeUtils.processorInstant(name)}.toInstant().toEpochMilli()" + else s"${DateDiff.utcInstant(name)}.toInstant().toEpochMilli()" + } + } + } + + /** The UTC day a temporal operand falls in, in days since the epoch. `Math.floor` is the one + * floor Painless allows -- `Math.floorDiv` is on no major's allow-list -- and it is exact here: + * a date's milliseconds are far below 2^53, and so is every quotient's distance to an integer. + */ + private[sql] def epochDay( + operand: PainlessScript, + ref: String, + context: Option[PainlessContext] + ): String = + s"Math.floor(${epochMillis(operand, ref, context)} / $FractionalDayMillis)" + + /** The instant a temporal operand denotes, in epoch milliseconds: a DATE is the instant its UTC + * day starts at, so two dates compare -- and move -- by their days. + */ + private[sql] def instantMillis( + operand: PainlessScript, + ref: String, + context: Option[PainlessContext] + ): String = + if (isCalendarDate(operand)) s"${epochDay(operand, ref, context)} * $FractionalDayMillis" + else epochMillis(operand, ref, context) + + /** A temporal operand of a COMPARISON -- a simple `CASE`, `NULLIF` -- as the instant it denotes: + * [[instantMillis]], the conversion date arithmetic and `GREATEST` / `LEAST` share, so a DATE is + * the instant its UTC day starts at and equals a TIMESTAMP exactly at midnight, as SQL engines + * promote it. A string literal the comparison READS as a temporal (`ComparisonRule.readingOf`) + * is its instant, computed here rather than parsed per document. + */ + private[sql] def comparedMillis( + operand: PainlessScript, + reading: Option[ComparisonRule.Reading], + ref: String, + context: Option[PainlessContext] + ): String = + reading match { + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case _ => instantMillis(operand, ref, context) + } + + private[math] def dateArithmetic(e: ArithmeticExpression): Option[DateArithmetic] = { + val l = operandOf(e.left, e.right) + val r = operandOf(e.right, e.left) + if (!temporal(l) && !temporal(r)) None + else + e.operator match { + case MULTIPLY | DIVIDE | MODULO => Some(refused(e)) + case ADD => + (l, r) match { + case (t, Days(fractional)) if temporal(t) => + Some(DaysShift(temporalLeft = true, t == CalendarDate, fractional)) + case (Days(fractional), t) if temporal(t) => + Some(DaysShift(temporalLeft = false, t == CalendarDate, fractional)) + case (NullLiteral, _) | (_, NullLiteral) => Some(NullOperand) + case (Undecided, _) | (_, Undecided) => None + case _ => Some(refused(e)) + } + case SUBTRACT => + (l, r) match { + case (CalendarDate, CalendarDate) => Some(DaysBetween) + case (a, b) if temporal(a) && temporal(b) => Some(FractionalDaysBetween) + case (t, Days(fractional)) if temporal(t) => + Some(DaysShift(temporalLeft = true, t == CalendarDate, fractional)) + case (NullLiteral, _) | (_, NullLiteral) => Some(NullOperand) + case (Undecided, _) | (_, Undecided) => None + case _ => Some(refused(e)) + } + } + } + + /** The type a refusal names an operand by: the one DESCRIBE shows for it. */ + private def typeName(o: PainlessScript): String = + if (bareNull(o)) SQLTypes.Null.typeId else declaredType(o).typeId + + private def refused(e: ArithmeticExpression): Refused = + Refused( + s"Type mismatch: operator ${e.operator.sql} cannot be applied to ${typeName(e.left)} and " + + s"${typeName(e.right)} in expression: ${e.sql.trim}; subtract two dates for the days between " + + "them, or add or subtract a number of days" + ) + + /** Every date-arithmetic type error in an expression, wherever an arithmetic sits in it. + * + * πŸ”΄ Asked of a SCHEMA-RESOLVED expression, as `NullIf.mismatchesOf` is: a computed column's + * resolved table (`schema.validateScriptReferences`) -- a statement asks every type rule at once + * (`SingleSearch.typeErrors`). Before a schema is attached a column's type is unknown, which + * these rules accept. `functionsOf` is the walk every such rule shares; an arithmetic nested + * directly in another is not a chain, so each arithmetic walks its own operands + * ([[ArithmeticExpression.dateArithmeticErrors]]). + */ + def typeErrorsOf(chain: FunctionChain): Seq[String] = + functionsOf(chain) + .collect { case a: ArithmeticExpression => a } + .flatMap(_.dateArithmeticErrors) + .distinct +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala index bf3006f36..8fcc58014 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/package.scala @@ -491,6 +491,37 @@ package object sql { case object Transform extends PainlessContextType } + /** The Elasticsearch major a Painless script is rendered for. + * + * This module is version-agnostic, and a script renders the same on every major -- except where + * a major hands a script something the others do not. The major then reaches the rendering the + * way issue #222 hands the search-body serializer to the bridge: the CLIENT module of each major + * puts its own `PainlessTarget` in implicit scope of the bridge's conversions, and they give it + * to every [[PainlessContext]] they create. With none in scope ([[PainlessTarget.Default]] -- a + * bridge's own tests, a script rendered outside a client), a script renders as it does for the + * majors from 7 on. + * + * @param elasticsearchMajor + * the major the client module talks to, `None` when no client module gave one + */ + final case class PainlessTarget(elasticsearchMajor: Option[Int]) { + + /** Does this major hand a QUERY script a `date` doc value as a `JodaCompatibleZonedDateTime`? + * Elasticsearch 6.8 does, and a comparison between one and a genuine `ZonedDateTime` fails + * there with `Class.cast` (`query.Expression.painless`); 7 and up hand a `ZonedDateTime`. + */ + def jodaCompatibleDates: Boolean = elasticsearchMajor.exists(_ < 7) + } + + object PainlessTarget { + + /** No client module gave a major: the rendering of the majors from 7 on. */ + val Default: PainlessTarget = PainlessTarget(None) + + /** The target of a client module talking to `elasticsearchMajor`. */ + def apply(elasticsearchMajor: Int): PainlessTarget = PainlessTarget(Some(elasticsearchMajor)) + } + /** Is a rendered Painless fragment a single EXPRESSION, i.e. may it be placed where an operand * goes? [[PainlessOperandForm.placeable]] is the one owner of that rule. * @@ -722,8 +753,13 @@ package object sql { /** Context for painless scripts * @param context * the context type + * @param target + * the Elasticsearch major the script is rendered for ([[PainlessTarget]]) */ - case class PainlessContext(context: PainlessContextType = PainlessContextType.Query) { + case class PainlessContext( + context: PainlessContextType = PainlessContextType.Query, + target: PainlessTarget = PainlessTarget.Default + ) { // List of parameter keys private[this] var _keys: collection.mutable.Seq[PainlessParam] = collection.mutable.Seq.empty @@ -1213,6 +1249,14 @@ package object sql { override def painless(context: Option[PainlessContext]): String = "null" override def nullable: Boolean = true override def baseType: SQLType = SQLTypes.Null + + /** `NULL` is ONE object, shared by every statement this JVM parses, so a cast must not write + * its type into it: `CAST(NULL AS INT)` -- a conversion's constructor casts its operand -- + * retyped EVERY later `NULL` as an INT, and `NULLIF(s, NULL)` was then refused as "KEYWORD + * with INT" (MEASURED in one test JVM). A typed NULL takes its type from its conversion, which + * already carries it; the bare `NULL` keeps none. + */ + override def cast(targetType: SQLType): SQLType = out } case object ParamValue extends Value[String](null) with TokenRegex { @@ -1334,9 +1378,14 @@ package object sql { for { _ <- from.validate() _ <- to.validate() - _ <- Validator.validateTypesMatching(from.out, to.out) } yield () } + + /** The two bounds must match -- asked once the column types are known, like every TYPE rule + * (the lead's ruling of 2026-10-05), never when the statement is parsed. + */ + override def typeError: Option[String] = + Validator.validateTypesMatching(from.out, to.out).left.toOption } case class LiteralFromTo(override val from: StringValue, override val to: StringValue) @@ -1547,6 +1596,303 @@ package object sql { def update(request: SingleSearch): Source } + /** The ONE comparison rule (the lead's ruling of 2026-10-05): what a comparison compares, and + * whether it may. Every comparison asks it -- `=`, `<>`, `<`, `<=`, `>`, `>=`, `BETWEEN`, `IN`, + * a simple `CASE x WHEN v`, `NULLIF` and a CASE condition, in every clause -- and so does the + * arithmetic that meets a string literal. + * + * - An operand is judged by the type it is DECLARED with ([[ComparisonRule.declaredTypeOf]]): + * what the user wrote, never the runtime collapse of every temporal to TIMESTAMP that + * `renderedTypeOf` answers. A DATE and a TIMESTAMP are comparable + * (`SQLTypeUtils.comparableTemporals`): the DATE is the instant its UTC day starts at. + * - An untyped string LITERAL is read by what it is compared with, as PostgreSQL reads one + * ([[ComparisonRule.readingOf]]): beside a number, the number it spells when that number is + * a value of the other's type -- beside a whole-number type, a whole number only; beside a + * temporal, the date or the timestamp it spells -- beside a DATE, a date-time literal is the + * date it names, as PostgreSQL's date input reads it; beside a BOOLEAN, the boolean it + * spells. Any other literal stays text, and the comparison is judged as such. + * + * A reading costs a parse, so each site computes a literal's reading ONCE per node, never per + * type question or per rendering: memoised in a field of the node, with no lock -- a reading is + * pure and immutable, so a race can only compute it twice, where a `lazy val`'s lock costs more + * than the reading on every fresh node. And a text that cannot spell a date never reaches a + * `java.time` parser and the exception it throws. + */ + object ComparisonRule { + + /** What a string literal is READ as. */ + sealed trait Reading { + def sqlType: SQLType + } + + /** The number a literal spells, as a Painless literal: a `long` for a whole number that fits + * one, the `double` it denotes otherwise. + */ + final case class NumberReading(painless: String, fractional: Boolean) extends Reading { + def sqlType: SQLType = if (fractional) SQLTypes.Double else SQLTypes.BigInt + } + + /** The date (`sqlType` DATE, `text` as `yyyy-MM-dd`) or the timestamp (`sqlType` TIMESTAMP, + * `text` an ISO date-time) a literal spells, and the instant it denotes in epoch milliseconds, + * UTC unless the literal names a zone; a date is the instant its day starts at. + */ + final case class TemporalReading(sqlType: SQLType, text: String, epochMillis: Long) + extends Reading + + /** The boolean a literal spells. */ + final case class BooleanReading(value: Boolean) extends Reading { + def sqlType: SQLType = SQLTypes.Boolean + } + + /** The text of an untyped STRING literal, bare or as the one function of a nameless identifier + * -- the parser wraps every operand in an identifier. + */ + def stringLiteral(token: Token): Option[String] = token match { + case s: StringValue => Some(s.value) + case i: Identifier if i.name.trim.isEmpty => + i.functions match { + case (s: StringValue) :: Nil => Some(s.value) + case _ => None + } + case _ => None + } + + /** The type an operand is DECLARED with: an aggregate is the metric it produces + * (`AggregateFunction.outputTypeOf`), anything else the type it reports (`reportedType`: a + * DATE column, a COALESCE of DATE columns, `d + 1` are DATEs). + */ + def declaredTypeOf(operand: Token): SQLType = operand match { + case i: Identifier if i.isAggregation => + AggregateFunction.outputTypeOf(i).getOrElse(i.reportedType) + case p: PainlessScript => p.reportedType + case other => other.out + } + + /** A whole number, as PostgreSQL's integer input reads one. */ + private val WholeNumber = "[+-]?\\d+".r + + /** A decimal number, with an optional exponent; `NaN` and `Infinity` are not numbers here. */ + private val DecimalNumber = "[+-]?(?:\\d+\\.?\\d*|\\.\\d+)(?:[eE][+-]?\\d+)?".r + + /** STRICT: `2026-02-30` is no date (PostgreSQL refuses it), where the default resolver would + * read it as February 28. + */ + private val IsoDate = java.time.format.DateTimeFormatter + .ofPattern("uuuu-MM-dd") + .withResolverStyle(java.time.format.ResolverStyle.STRICT) + + private val IsoDateTimeUtc = + java.time.format.DateTimeFormatter.ISO_DATE_TIME.withZone(java.time.ZoneOffset.UTC) + + private val DayMillis = 86400000L + + /** The number a text spells, if it spells one. */ + def numberOf(text: String): Option[NumberReading] = + text.trim match { + case s @ WholeNumber() => + Try(BigInt(s)).toOption.map { n => + if (n.isValidLong) NumberReading(s"${n}L", fractional = false) + else NumberReading(n.toDouble.toString, fractional = true) + } + case s @ DecimalNumber() => + Try(BigDecimal(s).toDouble).toOption + .filter(d => !d.isNaN && !d.isInfinite) + .map(d => NumberReading(d.toString, fractional = true)) + case _ => None + } + + /** The number a text spells beside an operand of the numeric type `against`, if it is a value + * of that type, as PostgreSQL's input for the type reads it (the lead's ruling of 2026-10-05): + * - beside a whole-number type (TINYINT, SMALLINT, INT, BIGINT), a whole number BIGINT can + * hold -- `'1.5'`, `'1.0'` and `'1e3'` are no integer (PostgreSQL: `invalid input syntax + * for type integer`), and a whole number past BIGINT's range is a value of no integer type + * (`out of range`); + * - beside DOUBLE, REAL or NUMERIC, any number. + * + * A literal that is no value of the type stays text, and the comparison or the arithmetic that + * reads it is refused as text against that type -- as PostgreSQL refuses `n = '1.5'` and `n + + * '1.5'` over an integer `n`. + */ + def numberOf(text: String, against: SQLNumeric): Option[NumberReading] = + numberOf(text) match { + case Some(number) if number.fractional && wholeNumberType(against) => None + case reading => reading + } + + private def wholeNumberType(t: SQLType): Boolean = t match { + case _: SQLTinyInt | _: SQLSmallInt | _: SQLInt | _: SQLBigInt => true + case _ => false + } + + /** Can the trimmed `text` spell a `uuuu-MM-dd` date (a `/` for the `-`)? A NECESSARY condition + * of the parser below, never a sufficient one: that parser reads a year of at least four + * digits (strict parsing), a sign at most, `-`, two digits, `-`, two digits -- ten characters + * or more of digits, `-`, `/` and `+`, and nothing else. A text that fails it never reaches + * the parser, whose refusal is an EXCEPTION: `'1'` beside a date cost two of them, 2 Β΅s. + */ + private def maySpellDate(text: String): Boolean = + text.length >= 10 && { + var i = 0 + var ok = true + while (ok && i < text.length) { + val c = text.charAt(i) + ok = (c >= '0' && c <= '9') || c == '-' || c == '/' || c == '+' + i += 1 + } + ok + } + + /** Can the trimmed `text` spell an ISO date-time (a space for the `T`)? A NECESSARY condition + * of the parser below: a year that starts with a digit or a sign, a date of ten characters or + * more, a `T` (any case, or the space read as one), and an `HH:mm` time at least -- sixteen + * characters, a `:`. + */ + private def maySpellDateTime(text: String): Boolean = + text.length >= 16 && { + val c = text.charAt(0) + (c >= '0' && c <= '9') || c == '-' || c == '+' + } && text.indexOf(':') > 0 && + (text.indexOf('T') > 0 || text.indexOf('t') > 0 || text.indexOf(' ') > 0) + + /** The date a text spells -- `yyyy-MM-dd`, a `/` accepted for the `-` -- if it spells one. */ + private def dateOf(text: String): Option[java.time.LocalDate] = { + val trimmed = text.trim + if (!maySpellDate(trimmed)) None + else Try(java.time.LocalDate.parse(trimmed.replace("/", "-"), IsoDate)).toOption + } + + /** The date-time an ISO text spells (a space accepted for the `T`), in UTC unless it names a + * zone, if it spells one. + */ + private def dateTimeOf(text: String): Option[java.time.ZonedDateTime] = { + val trimmed = text.trim + if (!maySpellDateTime(trimmed)) None + else + Try(java.time.ZonedDateTime.parse(trimmed.replace(" ", "T"), IsoDateTimeUtc)).toOption + } + + private def dateReading(date: java.time.LocalDate): TemporalReading = + TemporalReading(SQLTypes.Date, date.toString, date.toEpochDay * DayMillis) + + /** The temporal a text spells beside an operand of type `against`, read AS that operand's type, + * as PostgreSQL reads an untyped literal: + * - beside a DATE, the date it names -- a date-time text's date, as PostgreSQL's date input + * reads `'2024-01-31T10:00:00Z'` as 2024-01-31; + * - beside any other temporal, the TIMESTAMP it denotes -- a date-only text is the instant + * its UTC day starts at, so `ts = '2024-01-31'` holds at midnight only. Its `text` is then + * always a full ISO date-time, which is what the `CAST(… AS TIMESTAMP)` conversion parses. + */ + def temporalOf(text: String, against: SQLType): Option[TemporalReading] = { + val date = dateOf(text) + if (against == SQLTypes.Date) + date.orElse(dateTimeOf(text).map(_.toLocalDate)).map(dateReading) + else + date + .map { d => + TemporalReading(SQLTypes.Timestamp, s"${d}T00:00:00Z", d.toEpochDay * DayMillis) + } + .orElse { + dateTimeOf(text).map { zoned => + TemporalReading( + SQLTypes.Timestamp, + text.trim.replace(" ", "T"), + zoned.toInstant.toEpochMilli + ) + } + } + } + + /** The boolean a text spells, as PostgreSQL's boolean input reads one. */ + private def booleanOf(text: String): Option[BooleanReading] = + text.trim.toLowerCase match { + case "t" | "true" | "y" | "yes" | "on" | "1" => Some(BooleanReading(value = true)) + case "f" | "false" | "n" | "no" | "off" | "0" => Some(BooleanReading(value = false)) + case _ => None + } + + /** What `literal` is read as beside an operand DECLARED `against`, if it is an untyped string + * literal that spells a value of that type (`numberOf(text, against)` for a number); `None` + * otherwise -- it stays text. An operand of UNKNOWN type (no schema attached) reads no + * literal: nothing decides what it should be. + */ + def readingOf(literal: Token, against: => SQLType): Option[Reading] = + stringLiteral(literal).flatMap { text => + against match { + case n: SQLNumeric => numberOf(text, n) + case t: SQLTemporal if t != SQLTypes.Time && t != SQLTypes.Temporal => + temporalOf(text, t) + case SQLTypes.Temporal => temporalOf(text, SQLTypes.Timestamp) + case _: SQLBool => booleanOf(text) + case _ => None + } + } + + /** May two compared types be compared? */ + def comparable(left: SQLType, right: SQLType): Boolean = + SQLTypeUtils.matches(left, right) || SQLTypeUtils.comparableTemporals(left, right) + + /** The type an operand is compared AS: what it is read as beside the other operand + * ([[readingOf]], its `reading`), else the type it is declared with ([[declaredTypeOf]]), + * asked only then. + * + * A site passes the reading it holds -- computed ONCE per node, for its type rule and its + * rendering alike -- so a literal is never read a second time to be judged. + */ + def comparedType(reading: Option[Reading], operand: Token): SQLType = + reading match { + case Some(r) => r.sqlType + case None => declaredTypeOf(operand) + } + + /** The two compared types ([[comparedType]]), when they cannot be compared. */ + def mismatch(left: SQLType, right: SQLType): Option[(SQLType, SQLType)] = + if (comparable(left, right)) None else Some((left, right)) + + /** The Painless a NUMBER or a BOOLEAN reading renders: the literal itself. */ + def scalarPainless(reading: Reading): Option[String] = reading match { + case NumberReading(painless, _) => Some(painless) + case BooleanReading(value) => Some(value.toString) + case _ => None + } + + /** The VALUE a temporal reading denotes, as Painless that parses nothing: a DATE is the + * `LocalDate` of its day, a TIMESTAMP the UTC `ZonedDateTime` of its instant (core's + * number-to-TIMESTAMP conversion, `SQLTypeUtils.coerce`) -- the java.time types a `CAST('…' AS + * DATE)` / `CAST('…' AS TIMESTAMP)` of the same text renders, built from the reading when the + * statement is rendered rather than parsed from the literal on every document. + */ + def temporalPainless(reading: TemporalReading): String = + if (reading.sqlType == SQLTypes.Date) + s"LocalDate.ofEpochDay(${Math.floorDiv(reading.epochMillis, DayMillis)}L)" + else + SQLTypeUtils.coerce( + s"${reading.epochMillis}L", + SQLTypes.BigInt, + SQLTypes.Timestamp, + nullable = false, + None + ) + } + + /** The type a TYPE RULE reads for an operand once the column types are known: the one the operand + * RENDERS, never the one a resolved column reports through `out`. + * + * πŸ”΄ ONE derivation for every rule the lead's ruling of 2026-10-05 moved to the post-resolution + * seam, and for the arithmetic that types itself from its operands (#292). After the schema is + * attached `out` answers the COLUMN's type whatever function the operand wears -- `CAST(n AS + * INT)` over a KEYWORD column says KEYWORD, `COUNT(d)` says TIMESTAMP -- so a rule reading it + * would refuse `CAST(s AS INT) IN (1, 2)` and `HAVING COUNT(d) > 1`. So: + * - an AGGREGATE is the metric it produces (`AggregateFunction.outputTypeOf`); + * - any other identifier is what its chain renders (#382's `chainType`); + * - anything else -- a literal, a function, a nested arithmetic -- its own `out`. + */ + def renderedTypeOf(operand: Token): SQLType = operand match { + case i: Identifier if i.isAggregation => + AggregateFunction.outputTypeOf(i).getOrElse(i.chainType) + case i: Identifier => i.chainType + case other => other.out + } + sealed trait Identifier extends TokenValue with Source @@ -2236,7 +2582,8 @@ package object sql { * `runtimeType(declaration)`. A bare `DATE` column is reported as `DATE`, which is what the * user wrote and what DuckDB answers for the same column. */ - override def reportedLeafType: SQLType = col.map(_.dataType).getOrElse(super.reportedLeafType) + override def reportedLeafType: SQLType = + col.map(_.dataType).getOrElse(super.reportedLeafType) def update(request: SingleSearch): Identifier = query.Having.aliasedAggregate(this) match { diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala index 1ea0e14cc..c0042f9ce 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/parser/Validator.scala @@ -17,12 +17,12 @@ package app.softnetwork.elastic.sql.parser import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils} -import app.softnetwork.elastic.sql.function.{Function, FunctionN} +import app.softnetwork.elastic.sql.function.Function object Validator { def validateChain(functions: List[Function]): Either[String, Unit] = { - // validate function chain type compatibility + // every function of the chain, for its own syntax and structure functions match { case Nil => return Right(()) case _ => @@ -31,12 +31,10 @@ object Validator { case Nil => // ok case errors => return Left(errors.map { case Left(err) => err }.mkString("\n")) } - val funcs = functions.collect { case f: FunctionN[_, _] => f } - funcs.sliding(2).foreach { - case Seq(f1, f2) => - validateTypesMatching(f2.outputType, f1.inputType) - case _ => // ok - } + // ⚠️ No chain-link TYPE check here. One used to sit at this point and its result was + // DISCARDED (`foreach` over `validateTypesMatching`), so it never refused anything; it is not + // moved to the post-resolution seam either, because it never applied (the lead's ruling of + // 2026-10-05 moves the type checks that refuse, and adds none). Right(()) } @@ -50,7 +48,22 @@ object Validator { } trait Validation { + + /** The rules of SYNTAX and STRUCTURE, checked when the statement is parsed. */ def validate(): Either[String, Unit] = Right(()) + + /** This node's TYPE rule alone, if it breaks it: asked once the column types are known -- after + * the schema is attached (`SingleSearch.validateResolved`) -- and never at parse time. + * + * πŸ”΄ The lead's ruling of 2026-10-05: no refusal before the column types are known. At parse + * time every column is `Any`, so a type rule asked there either accepts what it cannot judge or + * refuses on a type it guessed: `d + 7 > CURRENT_DATE` was refused as BIGINT against DATE + * because `d + 7` folded `[ANY, BIGINT]` to BIGINT. Every TYPE refusal waits for the types, a + * refusal certain whatever the column's type (`x * CURRENT_DATE`) included; parsing checks + * syntax and structure only. With no schema at all, nothing is refused on types and + * Elasticsearch answers. + */ + def typeError: Option[String] = None } case class ValidationError(message: String) extends Exception(message) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala index c548263e2..7463e4d7b 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/Where.scala @@ -36,8 +36,8 @@ import app.softnetwork.elastic.sql.function.cond.{ } import app.softnetwork.elastic.sql.function.convert.Conversion import app.softnetwork.elastic.sql.function.geo.Distance -import app.softnetwork.elastic.sql.parser.Validator import app.softnetwork.elastic.sql.operator._ +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql._ import scala.annotation.tailrec @@ -85,6 +85,33 @@ sealed trait Criteria extends Updateable with PainlessScript { // operand of an equality, where the arm above sees only the equality itself. Found by review. }) ++ referencedIdentifiers.flatMap(Case.conditionsOf).flatMap(_.temporalComparisonErrors) + /** Every TYPE refusal of this criteria tree once the column types are known: each comparison's + * own rules ([[temporalComparisonErrors]]' TIME rule, then `Validation.typeError`), and every + * type rule of the functions its operands apply (`chainTypeErrors`), a CASE nested in an operand + * included. + * + * πŸ”΄ Asked of a SCHEMA-RESOLVED statement only (`SingleSearch.validateResolved`): the lead's + * ruling of 2026-10-05 -- no refusal before the column types are known. The same walk as + * [[temporalComparisonErrors]], so a criteria type that one reaches, this one reaches too. + * + * `scripted`: whether every comparison of this tree runs as a SCRIPT -- a CASE condition does, + * wherever the CASE sits. A `WHERE` / `HAVING` clause's own comparison of a BARE column with a + * literal runs as a query on the column's mapping instead, which Elasticsearch judges itself + * (`Expression.comparisonError`). + */ + def typeErrors: Seq[String] = typeErrors(scripted = false) + + def typeErrors(scripted: Boolean): Seq[String] = ((this match { + case Predicate(left, _, right, _, _) => left.typeErrors(scripted) ++ right.typeErrors(scripted) + case e: Expression => + e.temporalComparisonError.toSeq ++ e.comparisonError(scripted).toSeq + case relation: ElasticRelation => relation.criteria.typeErrors(scripted) + case _ => Nil + }) ++ referencedIdentifiers.flatMap(chainTypeErrors) ++ + referencedIdentifiers + .flatMap(Case.conditionsOf) + .flatMap(_.typeErrors(scripted = true))).distinct + def dependencies: Seq[Identifier] = this match { case Predicate(left, _, right, _, _) => left.dependencies ++ right.dependencies case c: Expression => c.dependencies @@ -780,7 +807,34 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def painlessOp: String = effectiveOperator.painless(None) def painlessValue(context: Option[PainlessContext]): String = - maybeValue.map(painlessOperand(_, context)).getOrElse("") + maybeValue match { + case Some(value) => readingPainless(literalReading, value, context) + case None => "" + } + + /** One operand as it is compared: a string literal by what the comparison reads it as + * ([[literalReading]] -- the number, the boolean, or the temporal it spells, the last converted + * exactly as `CAST('…' AS DATE)` / `CAST('…' AS TIMESTAMP)` converts it), anything else as it + * renders. + */ + protected def readingPainless( + reading: Option[ComparisonRule.Reading], + token: Token, + context: Option[PainlessContext] + ): String = + reading match { + case Some(t: ComparisonRule.TemporalReading) => + SQLTypeUtils.coerce( + s""""${t.text}"""", + SQLTypes.Varchar, + t.sqlType, + nullable = false, + context + ) + case Some(scalar) => + ComparisonRule.scalarPainless(scalar).getOrElse(painlessOperand(token, context)) + case None => painlessOperand(token, context) + } /** One operand's Painless rendering. A `Token` that is not a [[PainlessScript]] has only its SQL * spelling to offer -- which is why `BetweenExpr` must go through this and not `toString`: a @@ -909,11 +963,77 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { * right-hand type: this predicate's target type, and `check`'s per-type dispatch. */ protected def valueType: SQLType = - maybeValue match { - case Some(id: Identifier) => id.chainType - case Some(v: PainlessScript) => v.out - case Some(other) => other.out - case None => SQLTypes.Any + literalReading + .map(_.sqlType) + .getOrElse(maybeValue match { + case Some(id: Identifier) => id.chainType + case Some(v: PainlessScript) => v.out + case Some(other) => other.out + case None => SQLTypes.Any + }) + + /** What a STRING literal on the right is READ as -- the ONE comparison rule + * (`ComparisonRule.readingOf`), against the type the left operand is DECLARED with: `COALESCE(n, + * 0) = '3'` compares a number with the number 3, `d + 1 > '2024-01-01'` a date with a date, as + * SQL engines read an untyped literal by what it is compared with. + * + * A temporal is rendered by the very ` -> ` conversion a `CAST` uses + * ([[painlessValue]]) and reconciled with the left operand like any typed temporal; a number or + * a boolean renders as one. + * + * It decides only what a SCRIPT renders: a bare column compared with a literal in a `WHERE` runs + * as a query on the column's mapping, which never renders this comparison. + * + * A comparison that reads an AGGREGATE is a group filter, and a group filter reads every metric + * as a NUMBER -- a date metric as epoch milliseconds -- so while the metric's type is still + * unknown (before the schema is attached, or when none ever is) a literal is read there as the + * number or the instant it spells: nothing else can be compared with a metric. `HAVING MAX(n) > + * '4'` therefore renders the same comparison before and after resolution; the type rule judges + * it once the type is known. + * + * Computed ONCE per comparison and memoised (`ComparisonRule` says why with no lock): the type + * rule and every type question of the rendering ([[valueType]], [[comparisonTargetType]], + * `check`'s dispatch) ask it, and a reading is a parse. Attaching the schema COPIES the + * comparison (`update`), so the reading a resolved statement holds is read against the resolved + * types. + */ + protected def literalReading: Option[ComparisonRule.Reading] = { + var reading = _literalReading + if (reading eq null) { + reading = maybeValue.flatMap(readingFor) + _literalReading = reading + } + reading + } + + private[this] var _literalReading: Option[ComparisonRule.Reading] = null + + /** The type the left operand is DECLARED with (`ComparisonRule.declaredTypeOf`), which every + * literal of this comparison is read against and the type rule judges: computed ONCE, and + * memoised like [[literalReading]]. + */ + protected def operandDeclaredType: SQLType = { + var declared = _operandDeclaredType + if (declared eq null) { + declared = ComparisonRule.declaredTypeOf(identifier) + _operandDeclaredType = declared + } + declared + } + + private[this] var _operandDeclaredType: SQLType = null + + /** What `value` -- the right-hand side, a `BETWEEN` bound, an `IN` element -- is read as against + * this comparison's left operand ([[literalReading]]). + */ + protected def readingFor(value: Token): Option[ComparisonRule.Reading] = + ComparisonRule.stringLiteral(value).flatMap { text => + val declared = operandDeclaredType + ComparisonRule.readingOf(value, declared).orElse { + if (identifier.hasAggregation && declared.isUnknown) + ComparisonRule.numberOf(text).orElse(ComparisonRule.temporalOf(text, SQLTypes.Timestamp)) + else None + } } /** The JAVA type the RIGHT-hand side renders β€” what `Identifier.renderedType` answers for the @@ -1279,6 +1399,13 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { value: String, op: Operator ): String = { + // A bucket pipeline reads every metric as a NUMBER -- a date aggregate, and the alias of a + // per-group date calculation (`MAX(d) + 1 AS x ... HAVING x > ...`), arrive as epoch + // milliseconds -- and [[bucketPipelineCheck]] converts a temporal value to the same unit, so the + // two are compared as numbers. The `java.time` methods below would be called on a `Double` + // (`dynamic method [java.lang.Double, isAfter/1] not found`). + val readsMetric = + isAggregation || hasBucket || (context.isEmpty && (referencesBucketMetric || reducesDateMetrics)) op match { case _: ComparisonOperator => // πŸ”΄ Story BIDC-8 AD-9 β€” dispatch on the EFFECTIVE operator, i.e. with a `NOT` folded in, @@ -1300,7 +1427,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) < 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isBefore($value)" case _ => } @@ -1308,7 +1435,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) > 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isAfter($value)" case _ => } @@ -1331,7 +1458,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // spellings are otherwise identical for a `LocalTime`, nanoseconds included. case SQLTypes.Varchar | SQLTypes.Time => return s"$param.compareTo($value) == 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isEqual($value)" case _ => } @@ -1339,7 +1466,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar | SQLTypes.Time => return s"$param.compareTo($value) != 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isEqual($value) == false" case _ => } @@ -1347,7 +1474,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) >= 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isBefore($value) == false" case _ => } @@ -1355,7 +1482,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { valueType match { case SQLTypes.Varchar => return s"$param.compareTo($value) <= 0" - case _: SQLTemporal if !isAggregation && !hasBucket => + case _: SQLTemporal if !readsMetric => return s"$param.isAfter($value) == false" case _ => } @@ -1546,19 +1673,45 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { /** The comparison body of the bucket-pipeline rendering, `param` (= `params.`) against * the right-hand side. BETWEEN and IN, whose `painless` never goes through `check`, override it. */ - protected def bucketPipelineCheck(param: String): String = { + protected def bucketPipelineCheck(param: String): String = + check(None, param, bucketPipelineValue) + + /** The right-hand side as a group filter compares it with a metric: a NUMBER, epoch milliseconds + * for a temporal -- the unit a date metric arrives in. + */ + protected def bucketPipelineValue: String = literalReading match { + // a string literal read as a date or a timestamp ([[literalReading]]): its instant, computed + // once, here, rather than parsed per bucket + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case _ => bucketPipelineOperand + } + + private def bucketPipelineOperand: String = { val rhs = painlessValue(None) - val value = maybeValue match { - case Some(v) if operator.isInstanceOf[ComparisonOperator] && !v.isAggregation => + maybeValue match { + // A date-arithmetic value already renders epoch milliseconds here, the number a per-group + // date is (`HAVING MAX(d) > CAST('2024-03-01' AS DATE) - 30`): it is compared as it stands. + case Some(v) + if operator.isInstanceOf[ComparisonOperator] && !v.isAggregation && + !Expression.carriesArithmetic(v) => v.out match { - case SQLTypes.Date => s"$rhs.truncatedTo(ChronoUnit.DAYS).toInstant().toEpochMilli()" + // πŸ”΄ A DATE value is read through `getLong(ChronoField.EPOCH_DAY)`, which every + // `java.time` value with a date answers, whatever it renders: a `LocalDate` + // (`CAST('2024-01-01' AS DATE)`), which has neither `truncatedTo` nor `toInstant` -- + // `HAVING MAX(d) > CAST('2024-01-01' AS DATE)` failed to compile in Elasticsearch + // (`member method [java.time.LocalDate, truncatedTo/1] not found`, MEASURED on 8.18.3) + // -- or a UTC `ZonedDateTime` (date arithmetic, the clock). No `instanceof` and no NULL + // arm: the group-filter gate refuses both, and the value is a literal or the clock, + // never NULL. A TIMESTAMP value (a `ZonedDateTime`) was compared with the metric's + // number as it stood; it is converted like a DATETIME. + case SQLTypes.Date => s"(($rhs).getLong(ChronoField.EPOCH_DAY) * 86400000L)" case SQLTypes.Time => s"$rhs.truncatedTo(ChronoUnit.SECONDS).toInstant().toEpochMilli()" - case SQLTypes.DateTime => s"$rhs.toInstant().toEpochMilli()" - case _ => rhs + case SQLTypes.DateTime => s"$rhs.toInstant().toEpochMilli()" + case SQLTypes.Timestamp => s"($rhs).toInstant().toEpochMilli()" + case _ => rhs } case _ => rhs } - check(None, param, value) } /** True when this predicate reads a bucket-pipeline metric: an aggregate, or the alias of a @@ -1570,9 +1723,20 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { def referencesBucketMetric: Boolean = identifier.isAggregation || (identifier.hasAggregation && identifier.fieldAlias.isDefined) + /** `GREATEST` / `LEAST` over date AGGREGATES, inline in a group filter (`HAVING GREATEST(MAX(d), + * MAX(d2)) > CAST('2024-02-01' AS DATE)`): per group they render the epoch milliseconds of the + * date that wins (`NumericReducer`), the number a date aggregate is there, so they are compared + * as one -- the value converted to the same unit ([[bucketPipelineCheck]]). Compared as a + * `java.time` value, the group filter called `isAfter` on a `Double`. + */ + private def reducesDateMetrics: Boolean = + NumericReducer.overDates(identifier) && identifier.hasAggregation + override def painless(context: Option[PainlessContext]): String = { // A context-free rendering of an aggregate predicate is a bucket-pipeline rendering. if (context.isEmpty && referencesBucketMetric) return bucketPipelinePainless + if (context.isEmpty && reducesDateMetrics) + return s"$painlessNot(${bucketPipelineCheck(identifier.painless(None))})" val (chainRendering, innerLeft) = leftOperand(context) // The right-hand side is rendered ONCE: `painlessValue` can register a parameter on the context, @@ -1671,6 +1835,38 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { val leftNullable = readsDocumentField(Some(identifier)) val rightNullable = readsDocumentField(maybeValue) + /** πŸ”΄ Elasticsearch 6.8 hands a QUERY script a `date` doc value as a + * `JodaCompatibleZonedDateTime`, and its `isBefore` / `isAfter` / `isEqual` against a genuine + * `ZonedDateTime` -- which every function, every literal and date arithmetic produce -- fail + * the shard with `Class.cast` (MEASURED on 6.8.23, in both directions: `d > d2 - 1` and + * `CURRENT_DATE > d`). So each operand that READS a document and is compared as an instant is + * normalised to the `ZonedDateTime` it denotes, in UTC -- the conversion `DateDiff.utcInstant` + * already relies on (`withZoneSameInstant`), a no-op on a `ZonedDateTime` already in UTC. + * + * Only where the comparison is between INSTANTS (`comparisonTargetType` TIMESTAMP): an operand + * narrowed to a `LocalDate` or a `LocalTime` has no zone, and normalising one is what broke + * five comparisons in the first attempt at this. Only a QUERY script: an ingest processor + * parses its operands into `ZonedDateTime`s itself, and a view's transform reads its own + * index. The operand is a name or a call chain here, so the conversion is appended to it; it + * is never NULL there -- every route below tests it against NULL first. + * + * Only for Elasticsearch 6.8 ([[PainlessTarget.jodaCompatibleDates]], the major the client + * module gives the bridge): 7 and up hand a genuine `ZonedDateTime`, and there the conversion + * cost 2 to 4% of a per-document filter for nothing (MEASURED on 8.18.3), so the scripts of + * those majors stay what they were. + */ + val utcComparison: Boolean = + context.exists(c => !c.isProcessor && !c.isTransform && c.target.jodaCompatibleDates) && + valueType.isInstanceOf[SQLTemporal] && !isAggregation && !hasBucket && + comparisonTargetType == SQLTypes.Timestamp + def instantOperand(rendered: SQLType, readsDocument: Boolean): Boolean = + utcComparison && readsDocument && + (rendered == SQLTypes.Timestamp || rendered == SQLTypes.Date) + val leftUtc = instantOperand(operandRenderedType, leftNullable) + val rightUtc = instantOperand(valueRenderedType, rightNullable) + def utc(operand: String, normalise: Boolean): String = + if (normalise) s"$operand.withZoneSameInstant(ZoneId.of('Z'))" else operand + context match { case Some(ctx) => ctx.get(identifier) match { @@ -1705,9 +1901,9 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // `ABS(amount)` already used, and the reason those were never affected. case Some(p) if !rightNullable && p == chainRendering => if (identifier.nullable) - return s"$p == null ? false : $painlessNot(${check(context, p, rightPromoted)})" + return s"$p == null ? false : $painlessNot(${check(context, utc(p, leftUtc), utc(rightPromoted, rightUtc))})" else - return s"$painlessNot(${check(context, p, rightPromoted)})" + return s"$painlessNot(${check(context, utc(p, leftUtc), utc(rightPromoted, rightUtc))})" case _ => } case _ => @@ -1771,7 +1967,7 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { if (rightNullable) Some(s"$rightRef == null") else None ).flatten if (guards.nonEmpty && context.nonEmpty) - return s"(${guards.mkString(" || ")} ? false : ($painlessNot(${check(context, leftRef, promotedRight(rightRef, context))})))" + return s"(${guards.mkString(" || ")} ? false : ($painlessNot(${check(context, utc(leftRef, leftUtc), utc(promotedRight(rightRef, context), rightUtc))})))" // πŸ”΄ INVARIANT (story BIDC-8, review L-11): a statement sequence is legal ONLY with no // `PainlessContext`. `Criteria.painless` is spliced into larger expressions -- a `Predicate` // joins two renderings with `&&`/`||`, and a `CASE` condition captures one as @@ -1780,45 +1976,111 @@ sealed trait Expression extends FunctionChain with ElasticFilter with Criteria { // route is the HAVING / bucket-pipeline one, which is never spliced. Gated by // `PainlessOperandFormSpec`, which FAILS (14 shapes) if `bindLocal` stops hoisting. if (identifier.nullable) - return s"def left = $leftRef; left == null ? false : $painlessNot(${check(context, "left", rightPromoted)})" - s"$painlessNot${check(context, leftRef, rightPromoted)}" + return s"def left = $leftRef; left == null ? false : $painlessNot(${check(context, utc("left", leftUtc), utc(rightPromoted, rightUtc))})" + s"$painlessNot${check(context, utc(leftRef, leftUtc), utc(rightPromoted, rightUtc))}" } - override def validate(): Either[String, Unit] = { + /** STRUCTURE only: the two operands. The TYPE rule is [[typeError]], asked once the column types + * are known (the lead's ruling of 2026-10-05). + */ + override def validate(): Either[String, Unit] = for { _ <- identifier.validate() _ <- maybeValue match { - case Some(v) => - v.validate() match { - case Left(err) => Left(s"$err in expression: $this") - case Right(_) => - // πŸ”΄ Issue #384, item 4 β€” the types COMPARED are the ones the two operands END UP - // with, not the ones their COLUMNS have. `identifier.out` is the column's type, so - // this rule was wrong in BOTH directions: it accepted `CAST(ts AS TIME) > ts` - // (TIMESTAMP against TIMESTAMP, the cast invisible) and rejected - // `CAST(ts AS TIME) = CAST('07:00:00' AS TIME)` (TIMESTAMP against TIME, two - // `LocalTime`s that compare perfectly well). It is the same `out`-vs-`chainType` - // lie issue #367 fixed for EMISSION, still live in VALIDATION -- and `chainType` - // and `valueType` are the two derivations that already answer it for each side. - Validator.validateTypesMatching(identifier.chainType, valueType) match { - // πŸ”΄ ...and a comparison additionally accepts any two DATE-CARRYING temporals, - // because it is the one caller that RECONCILES them (issue #384, item 2). See - // `SQLTypeUtils.comparableTemporals` for why that allowance is NOT in `matches`. - case Left(_) if SQLTypeUtils.comparableTemporals(identifier.chainType, valueType) => - Right(()) - case Left(_) => - Left( - s"Type mismatch: '${identifier.chainType.typeId}' is not compatible with '${valueType.typeId}' in expression: $this" - ) - case Right(_) => Right(()) - } - } - case _ => Right(()) + case Some(v) => v.validate().left.map(err => s"$err in expression: $this") + case _ => Right(()) } } yield () + + /** The comparison's TYPE rule, asked once the column types are known, as a `WHERE` / `HAVING` + * clause asks it ([[comparisonError]]). + */ + override def typeError: Option[String] = comparisonError(scripted = false) + + /** The comparison's TYPE rule -- the ONE comparison rule (`ComparisonRule`, the lead's ruling of + * 2026-10-05) -- asked once the column types are known. + * + * πŸ”΄ It used to run when the statement was PARSED, where every column is `Any`: it refused `d + + * 7 > CURRENT_DATE` as BIGINT against DATE before `d` had a type, and could not see `s = n` at + * all. Asked after resolution it reads the types the two operands are DECLARED with, and + * accepts: + * - two matching types, and a DATE against a TIMESTAMP, judged on their declarations, never on + * the runtime collapse of every temporal to TIMESTAMP (`SQLTypeUtils.comparableTemporals`; a + * comparison RECONCILES them, issue #384); + * - a STRING literal read by what it is compared with: `COALESCE(n, 0) = '3'` is a number + * against 3, `d + 1 > '2024-01-01'` a date against a date (`ComparisonRule.readingOf`); + * - in a `WHERE` or `HAVING` clause's own comparison (`scripted = false`), any LITERAL against + * a BARE column: that comparison is a query Elasticsearch runs against the column's mapping, + * which coerces the literal (`n = '5'`, `d >= '2024-01-01'`) or refuses it itself, as it + * always has. A CASE condition is a script, so the rule judges it like any other comparison. + */ + def comparisonError(scripted: Boolean): Option[String] = + maybeValue.flatMap { value => + if (!scripted && Expression.literalAgainstBareColumn(identifier, value)) None + else + mismatchWith(value, literalReading).map { case (left, right) => + s"Type mismatch: '${left.typeId}' is not compatible with '${right.typeId}' in expression: $this" + } + } + + /** The ONE comparison rule (`ComparisonRule.mismatch`) between the left operand and `value` -- + * the right-hand side, a `BETWEEN` bound, an `IN` element -- judged with what this comparison + * already holds: `value`'s reading (`reading`, computed once) and the left operand's declared + * type ([[operandDeclaredType]]), so a literal is never read a second time to be judged. + */ + protected def mismatchWith( + value: Token, + reading: Option[ComparisonRule.Reading] + ): Option[(SQLType, SQLType)] = { + // the left operand is itself a string literal only in `'2024-01-01' < d`: then it is read + // beside `value`, as `value` is beside it + val left = + if (ComparisonRule.stringLiteral(identifier).isEmpty) operandDeclaredType + else + ComparisonRule + .readingOf(identifier, ComparisonRule.declaredTypeOf(value)) + .map(_.sqlType) + .getOrElse(operandDeclaredType) + ComparisonRule.mismatch(left, ComparisonRule.comparedType(reading, value)) } } +object Expression { + + /** Does a value carry arithmetic -- itself, or as a function of its chain? In a bucket pipeline, + * date arithmetic renders the epoch milliseconds a per-group date is. + */ + private[query] def carriesArithmetic(value: Token): Boolean = value match { + case _: ArithmeticExpression => true + case i: Identifier => i.functions.exists(_.isInstanceOf[ArithmeticExpression]) + case _ => false + } + + /** A column named with no function applied to it. */ + private[query] def bareColumn(token: Token): Boolean = token match { + case i: Identifier => i.name.trim.nonEmpty && i.functions.isEmpty + case _ => false + } + + /** A literal value, bare or as the one function of a nameless identifier. */ + private[query] def literal(token: Token): Boolean = token match { + case _: Value[_] => true + case i: Identifier => + i.name.trim.isEmpty && (i.functions match { + case (_: Value[_]) :: Nil => true + case _ => false + }) + case _ => false + } + + /** A LITERAL against a BARE column, on either side: the comparison runs as a query on the + * column's own mapping, and Elasticsearch coerces the literal or refuses it -- core never + * refused it, column types or not. + */ + private[query] def literalAgainstBareColumn(left: Token, right: Token): Boolean = + (bareColumn(left) && literal(right)) || (literal(left) && bareColumn(right)) +} + case class GenericExpression( identifier: Identifier, operator: ExpressionOperator, @@ -2082,20 +2344,69 @@ case class InExpr[R, +T <: Value[R]]( for { _ <- identifier.validate() _ <- values.validate() - _ <- { - val elementType = values.out match { - case a: SQLArray => a.elementType - case other => other - } - Validator - .validateTypesMatching(identifier.out, elementType) - .left - .map(_ => - s"Type mismatch: '${identifier.out.typeId}' is not compatible with '${elementType.typeId}' in expression: $this" - ) - } } yield () + /** Each element against the operand, by the ONE comparison rule (`ComparisonRule`), asked once + * the column types are known (the lead's ruling of 2026-10-05): `x IN (a, b)` is `x = a OR x = + * b`, so `COALESCE(n, 0) IN ('3', '10')` compares a number with two numbers, and `COALESCE(d, + * d2) IN ('2024-01-31', '2024-02-01')` a date with two dates. Judged on the operand's DECLARED + * type: `CAST(s AS INT) IN (1, 2)` over a KEYWORD `s` compares numbers. + * + * In a `WHERE` / `HAVING` clause's own IN over a BARE column (`scripted = false`), a `terms` + * query on the column's own mapping, which coerces the list or refuses it itself, as it always + * has. + */ + override def comparisonError(scripted: Boolean): Option[String] = + if (!scripted && Expression.bareColumn(identifier)) None + else + values.values + .zip(readings) + .flatMap { case (element, reading) => mismatchWith(element, reading) } + .headOption + .map { case (left, right) => + s"Type mismatch: '${left.typeId}' is not compatible with '${right.typeId}' in expression: $this" + } + + /** What each element is READ as (`readingFor`), computed ONCE per IN and memoised (like + * `literalReading`): the type rule, the type questions, both renderings and the group filter all + * ask it. + */ + private def readings: Seq[Option[ComparisonRule.Reading]] = { + var computed = _readings + if (computed eq null) { + computed = values.values.map(readingFor) + _readings = computed + } + computed + } + + private[this] var _readings: Seq[Option[ComparisonRule.Reading]] = null + + /** What every element is READ as (`readingFor`), when each one is a string literal the operand + * reads -- the list is then the numbers, the dates, the timestamps or the booleans they spell. + */ + private def elementReadings: Option[Seq[ComparisonRule.Reading]] = { + var computed = _elementReadings + if (computed eq null) { + computed = + if (values.values.isEmpty) None + else if (readings.forall(_.isDefined)) Some(readings.flatten) + else None + _elementReadings = computed + } + computed + } + + private[this] var _elementReadings: Option[Seq[ComparisonRule.Reading]] = null + + /** The type the list is compared IN when its elements are read ([[elementReadings]]), as the + * right-hand side of a single comparison is; the list's own type otherwise, as it always was. + */ + override protected def valueType: SQLType = + elementReadings + .map(rs => SQLTypeUtils.leastCommonSuperType(rs.map(_.sqlType).toList)) + .getOrElse(super.valueType) + /** `[v1, v2].contains()` -- the LIST owns `contains`, not the element. * * πŸ”΄ Story BIDC-8 (review L-8). The old override rendered @@ -2124,9 +2435,36 @@ case class InExpr[R, +T <: Value[R]]( // // A non-numeric list keeps `contains`: for strings `equals` is what IN means, and the rendering // is one term rather than N. - if (numericList) - values.values.map(v => s"$param == ${v.painless(context)}").mkString("(", " || ", ")") - else s"$value.contains($param)" + // + // A list of string literals the operand READS (`elementReadings`) is compared element by + // element, as the single comparison `x = ` compares: numbers and booleans with `==`, + // temporals through the shared per-type dispatch (`isEqual`), each element brought to the + // operand's java.time type first. `contains` asked a `ZonedDateTime` whether it EQUALS the + // string "2024-01-31" -- never, so `COALESCE(d, d2) IN ('2024-01-31', …)` matched no row. + elementReadings match { + case Some(readings) => + values.values + .zip(readings) + .map { + case (literal, t: ComparisonRule.TemporalReading) => + val rendered = readingPainless(Some(t), literal, context) + val target = comparisonTargetType + val element = + if ( + t.sqlType != target && target.isInstanceOf[SQLTemporal] && + !context.exists(_.isProcessor) + ) + SQLTypeUtils.coerce(rendered, t.sqlType, target, nullable = false, context) + else rendered + super.check(context, param, element, EQ) + case (element, reading) => + s"$param == ${readingPainless(Some(reading), element, context)}" + } + .mkString("(", " || ", ")") + case None if numericList => + values.values.map(v => s"$param == ${v.painless(context)}").mkString("(", " || ", ")") + case None => s"$value.contains($param)" + } /** True when every element of the list is a number β€” the case `List.contains` gets wrong. */ private def numericList: Boolean = @@ -2138,9 +2476,21 @@ case class InExpr[R, +T <: Value[R]]( // measured live, `MAX(age) IN (40, 50)` returned no bucket. Painless `==` promotes numerics and // uses `equals` for strings. The NOT is NOT rendered here: `IN` has no negated operator, so // `painlessNot` renders the `!` for it (review M-6) and `bucketPipelinePainless` already wraps - // this call in it -- negating twice was a double negation. + // this call in it -- negating twice was a double negation. A string element is the number, or + // the epoch milliseconds, it is read as -- a metric is a number (`readingFor`). override protected def bucketPipelineCheck(param: String): String = - values.values.map(v => s"$param == ${v.painless(None)}").mkString(" || ") + values.values + .zip(readings) + .map { case (v, reading) => + val element = reading match { + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case Some(scalar) => + ComparisonRule.scalarPainless(scalar).getOrElse(v.painless(None)) + case None => v.painless(None) + } + s"$param == $element" + } + .mkString(" || ") } @@ -2524,10 +2874,82 @@ case class BetweenExpr( for { _ <- identifier.validate() _ <- fromTo.validate() - _ <- Validator.validateTypesMatching(identifier.out, fromTo.out) } yield () } + /** Each bound against the operand, by the ONE comparison rule (`ComparisonRule`), asked once the + * column types are known (the lead's ruling of 2026-10-05): `x BETWEEN a AND b` is `x >= a AND x + * <= b`, so a bound is compared with the operand, never with the other bound -- `COALESCE(d, d2) + * BETWEEN '2024-01-01' AND '2024-02-01'` compares a date with two dates, and `d BETWEEN CAST(… + * AS DATE) AND CAST(… AS TIMESTAMP)` a date with a date and a timestamp. + * + * In a `WHERE` / `HAVING` clause's own BETWEEN over a BARE column (`scripted = false`), a + * `range` query on the column's own mapping, which coerces the bounds or refuses them itself, as + * it always has: only the bounds' own match is asked there. + */ + override def comparisonError(scripted: Boolean): Option[String] = + if (!scripted && Expression.bareColumn(identifier)) fromTo.typeError + else + Seq(fromTo.from, fromTo.to) + .flatMap(bound => mismatchWith(bound, boundReading(bound))) + .headOption + .map { case (left, right) => + s"Type mismatch: output '${left.typeId}' is not compatible with input '${right.typeId}'" + } + + /** What each bound is READ as (`readingFor`), computed ONCE per BETWEEN and memoised (like + * `literalReading`): the type rule, the type questions, both renderings and the group filter all + * ask it. + */ + private def boundReadings: (Option[ComparisonRule.Reading], Option[ComparisonRule.Reading]) = { + var computed = _boundReadings + if (computed eq null) { + computed = (readingFor(fromTo.from), readingFor(fromTo.to)) + _boundReadings = computed + } + computed + } + + private[this] var _boundReadings + : (Option[ComparisonRule.Reading], Option[ComparisonRule.Reading]) = + null + + private def boundReading(bound: Token): Option[ComparisonRule.Reading] = + if (bound eq fromTo.from) boundReadings._1 + else if (bound eq fromTo.to) boundReadings._2 + else readingFor(bound) + + /** The SQL type of a bound as it is compared: its reading, or its own type -- the question + * `valueType` answers for the right-hand side of a single comparison. + */ + private def boundType(bound: Token): SQLType = + boundReading(bound) + .map(_.sqlType) + .getOrElse(bound match { + case id: Identifier => id.chainType + case other => other.out + }) + + /** The JAVA type a bound renders -- the question `valueRenderedType` answers for a single one. */ + private def boundRenderedType(bound: Token): SQLType = + boundReading(bound) + .map(_.sqlType) + .getOrElse(bound match { + case id: Identifier => id.renderedType + case other => other.out + }) + + /** The type the bounds are compared IN -- what the shared dispatch ([[check]]) and the operand's + * target type ([[comparisonTargetType]]) read for the right-hand side of a single comparison. + */ + override protected def valueType: SQLType = + SQLTypeUtils.leastCommonSuperType(List(boundType(fromTo.from), boundType(fromTo.to))) + + override protected def valueRenderedType: SQLType = + SQLTypeUtils.leastCommonSuperType( + List(boundRenderedType(fromTo.from), boundRenderedType(fromTo.to)) + ) + /** Two comparisons, `param >= from && param <= to`, each rendered by the SHARED per-type * dispatch. * @@ -2539,6 +2961,13 @@ case class BetweenExpr( * [[Expression.check]] knows. Overriding `check` puts BETWEEN on the ONE guarded emission path * and reuses that dispatch via the explicit `GE` / `LE` operators. The NOT is rendered by * `painlessNot` (`BETWEEN` has no negated operator). + * + * Each bound is rendered as the comparison reads it (a string literal by its reading: the + * number, or the temporal, it spells) and, like the right-hand side of a single comparison, + * brought to the operand's java.time type when the two differ: a DATE bound beside a TIMESTAMP + * operand is the instant its day starts at. Compared as rendered, a string bound was compared as + * TEXT (`String.valueOf().compareTo("2024-02-01")`, right only while the two spellings + * happened to sort alike) and a `LocalDate` bound failed against a `ZonedDateTime`. */ override protected def check( context: Option[PainlessContext], @@ -2546,17 +2975,35 @@ case class BetweenExpr( value: String, op: Operator ): String = { - val from = painlessOperand(fromTo.from, context) - val to = painlessOperand(fromTo.to, context) + def bound(b: Token): String = { + val rendered = readingPainless(boundReading(b), b, context) + val from = boundRenderedType(b) + val to = comparisonTargetType + if ( + from != to && from.isInstanceOf[SQLTemporal] && to.isInstanceOf[SQLTemporal] && + !context.exists(_.isProcessor) + ) + SQLTypeUtils.coerce(rendered, from, to, nullable = false, context) + else rendered + } + val from = bound(fromTo.from) + val to = bound(fromTo.to) s"(${super.check(context, param, from, GE)} && ${super.check(context, param, to, LE)})" } // The guarded bucket form of ` BETWEEN a AND b` -- two comparisons, not a chained // `a <= p <= b` (Painless rejects `boolean <= int`). The NOT is NOT rendered here: `BETWEEN` has // no negated operator, so `painlessNot` renders the `!` (review M-6) and - // `bucketPipelinePainless` already wraps this call in it. - override protected def bucketPipelineCheck(param: String): String = - s"$param >= ${fromTo.from} && $param <= ${fromTo.to}" + // `bucketPipelinePainless` already wraps this call in it. A string bound is the number, or the + // epoch milliseconds, it is read as -- a metric is a number (`readingFor`). + override protected def bucketPipelineCheck(param: String): String = { + def bound(b: Token): String = boundReading(b) match { + case Some(t: ComparisonRule.TemporalReading) => s"${t.epochMillis}L" + case Some(scalar) => ComparisonRule.scalarPainless(scalar).getOrElse(b.toString) + case None => b.toString + } + s"$param >= ${bound(fromTo.from)} && $param <= ${bound(fromTo.to)}" + } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala index f511370e2..537667c4a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/query/package.scala @@ -18,6 +18,7 @@ package app.softnetwork.elastic.sql import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypeUtils, SQLTypes} import app.softnetwork.elastic.sql.operator.{AND, OR, SetOperator, UNION} +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.schema.{ sqlConfig, validateScriptReferences, @@ -34,8 +35,8 @@ import app.softnetwork.elastic.sql.schema.{ Table => Schema, TableType } -import app.softnetwork.elastic.sql.function.{FunctionChain, FunctionUtils} -import app.softnetwork.elastic.sql.function.cond.{Case, NullIf} +import app.softnetwork.elastic.sql.function.{BinaryFunction, FunctionChain, FunctionUtils} +import app.softnetwork.elastic.sql.function.cond.{functionsOf, Case, NumericReducer} import app.softnetwork.elastic.sql.function.aggregate.WindowFunction import app.softnetwork.elastic.sql.function.time.DateDiff import app.softnetwork.elastic.sql.policy.{EnrichPolicy, EnrichPolicyType} @@ -63,6 +64,26 @@ import scala.util.Try package object query { + /** Every TYPE refusal of the functions reachable from `chain` (`functionsOf`), each one's own + * `Validation.typeError`, an arithmetic nested directly in another included -- it is an operand, + * not a chain, so `functionsOf` does not reach it. + * + * πŸ”΄ Asked of a SCHEMA-RESOLVED expression only: the lead's ruling of 2026-10-05, no refusal + * before the column types are known. ONE walk for every function rule moved off `validate()` + * (`ArithmeticExpression`, `NULLIF`, `CASE`, `GREATEST` / `LEAST`, the `INTERVAL` family). + */ + private[elastic] def chainTypeErrors(chain: FunctionChain): Seq[String] = { + def arithmetic(a: ArithmeticExpression): Seq[String] = + a.typeError.toSeq ++ a.args.flatMap { + case nested: ArithmeticExpression => arithmetic(nested) + case _ => Nil + } + functionsOf(chain).flatMap { + case a: ArithmeticExpression => arithmetic(a) + case f => f.typeError.toSeq + }.distinct + } + /** The `SingleSearch`es a statement EMBEDS β€” the ONE place these arms are spelled. * * Every closure-shaped question ("does this statement need the relational engine?", "does it @@ -1025,7 +1046,7 @@ package object query { case relation: ElasticRelation => walk(relation.criteria) case leaf => leaf.referencedIdentifiers - .filter(id => !id.isAggregation && id.hasAggregation) + .filter(SingleSearch.readsPerGroupCalculation) .map(leaf -> _) } having.flatMap(_.criteria).toSeq.flatMap(walk) @@ -1763,12 +1784,11 @@ package object query { * (`SearchApi.resolveWithSchema`), which is the same seam #306 uses to attach the schema in * the first place. * - * ⚠️ It is NOT a second `validate()`. Running the whole of `validate()` against resolved types - * would surface every pre-existing disagreement between a column's DECLARED type and its - * chain's β€” including `CAST(ts AS TIME) = CAST('07:00:00' AS TIME)`, which works today and - * which `Expression.validate`'s `identifier.out` (the COLUMN's type, not the chain's) already - * rejects when it is asked after resolution. This method carries exactly the rules that were - * measured, and each one names the shape it refuses. + * ⚠️ It is NOT a second `validate()`: `validate()` checks syntax and structure only, and this + * method asks the TYPE rules ([[typeErrors]]). Each of them reads the type an operand RENDERS + * (`renderedTypeOf`: the chain's, or an aggregate's own output type), never a column's + * declared type alone -- `CAST(ts AS TIME) = CAST('07:00:00' AS TIME)` compares a TIME with a + * TIME -- and each one names the shape it refuses. */ def validateResolved(): Either[String, Unit] = // πŸ”΄ Issue #389 / F4 -- the representability gate runs AGAIN here, on the RESOLVED statement. @@ -1783,15 +1803,33 @@ package object query { // which is issue #250's family. That throw stays as the last-resort invariant; this is what // makes it unreachable in practice. having.map(_.unrepresentable).getOrElse(Nil).headOption.map(u => Left(u.message)).getOrElse { - ((where.flatMap(_.criteria).toSeq ++ having.flatMap(_.criteria).toSeq ++ - select.fields.flatMap(f => Case.conditionsOf(f.identifier)) ++ + typeErrors.headOption.map(Left(_)).getOrElse(Right(())) + } + + /** Every TYPE refusal of this RESOLVED statement, by name, in clause order. + * + * πŸ”΄ The lead's ruling of 2026-10-05: no refusal before the column types are known. Every TYPE + * rule that `validate()` used to apply when the statement was parsed -- where every column is + * `Any` -- is asked here instead, on the real types, alongside the rules that always needed + * them: a comparison's ([[Criteria.typeErrors]]: TIME against a date, the operand match, `IN`, + * `BETWEEN`), and every function's ([[chainTypeErrors]]: date arithmetic and the operand + * match, `NULLIF`, `CASE`, `GREATEST` / `LEAST`, `INTERVAL`). + * + * With NO schema attached nothing calls this, and nothing is refused on types: Elasticsearch + * answers. `SearchApi.resolveWithSchema` calls it through [[validateResolved]]; so does a + * `CREATE TABLE ... AS SELECT` before it creates -- or, with `OR REPLACE`, deletes -- its + * target. + */ + def typeErrors: Seq[String] = + // A WHERE / HAVING clause's own comparison may be a QUERY on a column's mapping; a CASE + // condition is always a script (`Criteria.typeErrors`). + ((where.flatMap(_.criteria).toSeq ++ having.flatMap(_.criteria).toSeq) + .flatMap(_.typeErrors(scripted = false)) ++ + (select.fields.flatMap(f => Case.conditionsOf(f.identifier)) ++ orderBy.toSeq.flatMap(_.sorts.flatMap(s => Case.conditionsOf(s.field))) ++ groupBy.toSeq.flatMap(_.buckets.flatMap(b => Case.conditionsOf(b.identifier)))) - .flatMap(_.temporalComparisonErrors) ++ - scriptedExpressions.flatMap(NullIf.mismatchesOf)).headOption - .map(Left(_)) - .getOrElse(Right(())) - } + .flatMap(_.typeErrors(scripted = true)) ++ + scriptedExpressions.flatMap(chainTypeErrors)).distinct /** Every expression of this statement that can be emitted as Painless, as the chain it is. * @@ -1849,11 +1887,12 @@ package object query { // Arithmetic over aggregates written INLINE in HAVING (`HAVING MAX(x) - MIN(x) > 3`) has no // aggregation to read from and was silently dropped. Alias it in SELECT and reference the // alias (`... AS d ... HAVING d > 3`), which IS supported (Having.resolveAggregateAliases). + // `GREATEST` / `LEAST` inline are no arithmetic (`SingleSearch.readsPerGroupCalculation`). having .flatMap(_.criteria) .map(_.referencedIdentifiers) .getOrElse(Nil) - .find(id => !id.isAggregation && id.hasAggregation && id.fieldAlias.isEmpty) match { + .find(id => id.fieldAlias.isEmpty && SingleSearch.readsPerGroupCalculation(id)) match { case Some(id) => Left( s"HAVING cannot combine aggregates arithmetically inline (${id.sql}); alias the expression in SELECT and reference the alias" @@ -2330,6 +2369,32 @@ package object query { object SingleSearch { + /** Does a HAVING reference read a per-group calculation the way an arithmetic over aggregates + * is read? The ONE predicate the inline-arithmetic refusal (`validate()`) and a materialized + * view's rule 4 (`havingBucketScriptRefs`) share (#292). + * + * - ALIASED (`SELECT GREATEST(MAX(a), MAX(b)) AS m ... HAVING m > 1`, once + * `Having.resolveAggregateAliases` has put the item in place), it reads the SELECT item's + * `bucket_script` by name: any expression over aggregates, `GREATEST` / `LEAST` over them + * included. + * - INLINE, it is an ARITHMETIC over aggregates, the shape the group filter cannot read. + * `GREATEST` / `LEAST` report the aggregates of their arguments + * (`NumericReducer.hasAggregation`, so that a SELECT computes them per group), but inline + * they are a function over aggregates, which the group filter reads through its operands + * (issue #389) -- alone or under an arithmetic, exactly as before they reported any. + */ + private[query] def readsPerGroupCalculation(id: Identifier): Boolean = { + def carriesAggregate(token: Token): Boolean = token match { + case _: NumericReducer => false + case binary: BinaryFunction[_, _, _] => + carriesAggregate(binary.left) || carriesAggregate(binary.right) + case i: Identifier => i.isAggregation || i.functions.exists(carriesAggregate) + case other => other.hasAggregation + } + !id.isAggregation && + (if (id.fieldAlias.isDefined) id.hasAggregation else id.functions.exists(carriesAggregate)) + } + /** Name of the synthetic single-bucket aggregation the bridge emits for a `HAVING` with no * `GROUP BY` (see [[SingleSearch.wholeTableHaving]]). It exists only so that the * `bucket_selector` has a multi-bucket parent to hang from -- Elasticsearch rejects one inside @@ -2516,7 +2581,9 @@ package object query { case None => Right(()) } _ <- MultiSearch.branchArity(requests) - _ <- MultiSearch.branchTypes(requests) + // ⚠️ No `branchTypes` here: it is a TYPE rule, asked once each branch's schema is attached + // (`SearchApi.resolveWithSchema(MultiSearch)`), never when the statement is parsed (the + // lead's ruling of 2026-10-05). } yield () } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala index c1fd29936..17cc09a09 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/schema/package.scala @@ -24,7 +24,9 @@ import app.softnetwork.elastic.sql.function.{ FunctionN, FunctionWithIdentifier } +import app.softnetwork.elastic.sql.function.convert import app.softnetwork.elastic.sql.function.cond.{Case, NullIf} +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression import app.softnetwork.elastic.sql.parser.Parser import app.softnetwork.elastic.sql.query._ import app.softnetwork.elastic.sql.serialization._ @@ -788,6 +790,78 @@ package object schema { .filter(path => projection.find(path).isEmpty) .distinct + /** The schema an index PATTERN or a comma list (`FROM logs-*`) answers with, so it is typed as + * the single index is: the union of its indices' columns, merged FIELD BY FIELD (the lead's + * ruling of 2026-10-05). + * + * - A field every index that maps it maps the SAME way -- the same type (a field no index + * declared is typed at the declaration seam, `schema.IndexField`: an undeclared `date` is a + * TIMESTAMP, like a declared one), the same date `format` (the one mapping option a + * statement's meaning depends on: a temporal literal is read against it) -- keeps that type, + * whatever the indices disagree on elsewhere. A field one index maps and another does not is + * the one index's. + * - A field two indices map DIFFERENTLY gets no type (`SQLTypes.Any`): a type cannot be chosen + * between them, so nothing is refused on it and Elasticsearch answers for it, as it does + * with no schema at all (the ruling's rule 4). It stays IN the schema: the name exists, and + * a subquery's scope must know it does. + * - Sub-fields (multi-fields, object properties) merge by the same rule, under their parent. + * + * πŸ”΄ Found by review, MEASURED on Elasticsearch 8.18.3: `SELECT g, MAX(d) - MIN(d) FROM zwild_* + * GROUP BY g` answered 2592000000 (milliseconds) where the single index answered 30 (days) -- + * with no schema attached a pattern was typed `Any` throughout, and the date-arithmetic rules + * could not see a date. A first merge attached NOTHING as soon as the indices disagreed on one + * field, which left every other field of such a pattern in that state. + * + * @return + * the merged schema, and every field the indices disagree on, by path, with the types they + * give it + */ + private[elastic] def mergedSchema( + name: String, + members: Seq[(String, Schema)] + ): (Schema, Seq[String]) = { + def format(c: Column): Option[Value[_]] = c.options.get("format") + def sameMapping(a: Column, b: Column): Boolean = + a.dataType == b.dataType && format(a) == format(b) + val conflicts = scala.collection.mutable.LinkedHashMap.empty[String, Set[String]] + def merge(into: List[Column], columns: List[Column], parent: String): List[Column] = { + val merged = scala.collection.mutable.LinkedHashMap(into.map(c => c.name -> c): _*) + columns.foreach { column => + val path = s"$parent${column.name}" + merged.get(column.name) match { + case None => merged.put(column.name, column) + case Some(first) => + val multiFields = merge(first.multiFields, column.multiFields, s"$path.") + if (sameMapping(first, column)) + merged.put(column.name, first.copy(multiFields = multiFields)) + else { + conflicts.put( + path, + conflicts.getOrElse(path, Set(first.dataType.typeId)) + column.dataType.typeId + ) + merged.put( + column.name, + first.copy( + dataType = SQLTypes.Any, + options = first.options - "format", + multiFields = multiFields + ) + ) + } + } + } + merged.values.toList + } + val columns = + members.foldLeft(List.empty[Column]) { case (acc, (_, schema)) => + merge(acc, schema.columns, "") + } + val disagreements = conflicts.toSeq.map { case (path, types) => + s"'$path' (${types.filterNot(_ == SQLTypes.Any.typeId).toSeq.sorted.mkString(", ")})" + } + (Table(name, columns = columns.sortBy(_.name)).update(), disagreements) + } + /** Reject a computed column whose expression reads a column the table does not declare, or hands * a column to a function that cannot take its declared type. * @@ -874,6 +948,47 @@ package object schema { case errors => return Left(errors.mkString("; ")) } + // Date arithmetic refused by name -- `d * 2`, `d + d2`, `2 - d` -- for the same two reasons as + // the NULLIF rule above: it needs the RESOLVED table, and no reference population, so it sits + // above the `isRegular` carve-out too. An ingest script would otherwise fail per document, and + // `ignore_failure` would swallow it into an absent column. + def arithmeticErrors(column: Column): Seq[String] = + column.script + .filterNot(_.materialized) + .toSeq + .flatMap(_.validationExpr) + .flatMap { + case chain: FunctionChain => ArithmeticExpression.typeErrorsOf(chain) + case _ => Nil + } + .map(reason => s"Column '${column.path}': $reason") ++ + column.multiFields.flatMap(arithmeticErrors) + + schema.columns.flatMap(arithmeticErrors).distinct match { + case Nil => + case errors => return Left(errors.mkString("; ")) + } + + // A number converted to a DATE -- `(d - d2)::DATE`, `CAST(n AS DATE)` -- refused by name for + // the same reasons, as a query refuses it (`Conversion.typeError`): the script would store the + // number itself in a DATE column. + def conversionErrors(column: Column): Seq[String] = + column.script + .filterNot(_.materialized) + .toSeq + .flatMap(_.validationExpr) + .flatMap { + case chain: FunctionChain => convert.typeErrorsOf(chain) + case _ => Nil + } + .map(reason => s"Column '${column.path}': $reason") ++ + column.multiFields.flatMap(conversionErrors) + + schema.columns.flatMap(conversionErrors).distinct match { + case Nil => + case errors => return Left(errors.mkString("; ")) + } + if (!schema.isRegular) return Right(()) def columnErrors(column: Column): Seq[String] = @@ -898,6 +1013,14 @@ package object schema { case Some(referenced) => consumer.collect { case fn: FunctionN[_, _] => fn }.toSeq.flatMap { fn => if (acceptableOperand(referenced.dataType, fn.inputType)) Nil + // A DATE, TIMESTAMP or DATETIME operand of `+` / `-` is date arithmetic, which the + // rule above judges on BOTH operands -- a NUMERIC input type cannot say whether + // `d - d2` is legal. A TIME is outside those rules and keeps this check. + else if ( + fn.isInstanceOf[ArithmeticExpression] && referenced.dataType.isTemporal && + referenced.dataType != SQLTypes.Time + ) + Nil else Seq( s"Column '${column.path}' applies ${fn.sql} to '${id.path}', which is " + diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala index d56ca35be..996cc696e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/type/SQLTypeUtils.scala @@ -25,6 +25,7 @@ import app.softnetwork.elastic.sql.{ PainlessScript } import app.softnetwork.elastic.sql.`type`.SQLTypes._ +import app.softnetwork.elastic.sql.function.time.DateDiff object SQLTypeUtils { @@ -422,24 +423,31 @@ object SQLTypeUtils { s"((${painlessType(t)}) $expr)" // ---- NUMERIC <-> TEMPORAL ---- + // A number cast to a TIMESTAMP is the instant that many milliseconds from the epoch -- + // core's rule, and it holds for EVERY numeric source, not only a BIGINT one: with the + // BIGINT arm alone `CAST(x AS TIMESTAMP)` over a DOUBLE, and `(ts - ts2)::TIMESTAMP` (a + // DOUBLE number of days), fell to the identity fallback and answered the number itself -- + // a cast that silently did nothing. A fractional number of milliseconds is truncated, as + // every other fractional-to-integral cast here is. case (SQLTypes.BigInt, SQLTypes.Timestamp | SQLTypes.DateTime) => s"Instant.ofEpochMilli($expr).atZone(ZoneId.of('Z'))" + case ( + SQLTypes.TinyInt | SQLTypes.SmallInt | SQLTypes.Int, + SQLTypes.Timestamp | SQLTypes.DateTime + ) => + s"Instant.ofEpochMilli(((long) $expr)).atZone(ZoneId.of('Z'))" + case ( + SQLTypes.Double | SQLTypes.Real | SQLTypes.Numeric, + SQLTypes.Timestamp | SQLTypes.DateTime + ) => + s"Instant.ofEpochMilli(((long) $expr)).atZone(ZoneId.of('Z'))" case (SQLTypes.Timestamp | SQLTypes.DateTime, SQLTypes.BigInt) => s"$expr.toInstant().toEpochMilli()" // ---- BOOLEAN -> NUMERIC ---- - case (SQLTypes.Boolean, SQLTypes.Numeric | SQLTypes.Int) => - s"($expr ? 1 : 0)" - case (SQLTypes.Boolean, SQLTypes.BigInt) => - s"($expr ? 1L : 0L)" - case (SQLTypes.Boolean, SQLTypes.Double) => - s"($expr ? 1.0 : 0.0)" - case (SQLTypes.Boolean, SQLTypes.Real) => - s"($expr ? 1.0f : 0.0f)" - case (SQLTypes.Boolean, SQLTypes.SmallInt) => - s"(short)($expr ? 1 : 0)" - case (SQLTypes.Boolean, SQLTypes.TinyInt) => - s"(byte)($expr ? 1 : 0)" + case (SQLTypes.Boolean, numeric) if booleanNumerals(numeric).isDefined => + val BooleanNumerals(one, zero, narrowing) = booleanNumerals(numeric).get + s"${narrowing.getOrElse("")}($expr ? $one : $zero)" // ---- -> BOOLEAN ---- // πŸ”΄ There were seven arms FROM boolean and NONE to it, so every `(_, Boolean)` pair fell @@ -719,10 +727,19 @@ object SQLTypeUtils { val parsed = declared match { // A DATE is written date-only, and `ZonedDateTime.parse` REFUSES a date with no time // (measured), so it is parsed as a `LocalDate` and then given the UTC start of day. + // + // πŸ”΄ ...unless it is longer than a date: Elasticsearch accepts a date-time string into a + // `date` field declared DATE, and the strict `yyyy-MM-dd` parse threw on it -- which a + // computed column's `ignore_failure: true` stored as NULL (`CASE WHEN .. THEN d ELSE d2 END - + // d`, MEASURED on Elasticsearch 8.18.3). Such a value is the instant it spells, given its UTC + // day, as `processorInstant` reads it and as the query venue's doc value is. case SQLTypes.Date => + val dateTime = coerce(expr, SQLTypes.Varchar, SQLTypes.Timestamp, nullable = false, None) Some( + s"($expr.length() > 10 ? $dateTime.withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + coerce(expr, SQLTypes.Varchar, SQLTypes.Date, nullable = false, None) + - ".atStartOfDay(ZoneId.of('Z'))" + ".atStartOfDay(ZoneId.of('Z')))" ) case SQLTypes.DateTime | SQLTypes.Timestamp | SQLTypes.Temporal => Some(coerce(expr, SQLTypes.Varchar, SQLTypes.Timestamp, nullable = false, None)) @@ -734,6 +751,31 @@ object SQLTypeUtils { parsed.map(p => s"($expr instanceof String ? $p : $epoch)") } + /** An INGEST operand as the UTC instant it denotes, decided by its RUNTIME class: a string is + * parsed by its shape -- a date at the start of its UTC day, a date-time as the instant it + * spells, through the very ` -> ` conversions a `CAST` uses -- a number is + * epoch milliseconds, and anything else is a temporal a function already made + * (`DateDiff.utcInstant`). + * + * πŸ”΄ Found by review, measured on Elasticsearch 8.18.3: an operand that is not a bare column + * (`COALESCE(d, d2) - d2`, `NULLIF(d, d2) - d2`) holds the raw JSON string, and reading it as a + * temporal threw on every document -- which a computed column's `ignore_failure: true` stored as + * NULL on every row. And a DATE column holding a date-time string (Elasticsearch accepts one) + * failed the strict `yyyy-MM-dd` parse the same way. Both now read the instant the value spells, + * as the query venue's doc value does, and a DATE operand is then floored to its UTC day. + * + * `ref` is read up to five times: it is a name. + */ + private[sql] def processorInstant(ref: String): String = { + val dateTime = coerce(ref, SQLTypes.Varchar, SQLTypes.Timestamp, nullable = false, None) + val date = + coerce(ref, SQLTypes.Varchar, SQLTypes.Date, nullable = false, None) + + ".atStartOfDay(ZoneId.of('Z'))" + s"($ref == null ? null : $ref instanceof String ? ($ref.length() > 10 ? $dateTime : $date) : " + + s"$ref instanceof Number ? Instant.ofEpochMilli($ref).atZone(ZoneId.of('Z')) : " + + s"${DateDiff.utcInstant(ref)})" + } + private val painlessPrimitives: Set[String] = Set("byte", "short", "int", "long", "float", "double", "boolean", "char") @@ -757,6 +799,37 @@ object SQLTypeUtils { case _ => false } + /** What a BOOLEAN is as a number of type `to`: TRUE's numeral, FALSE's, and the narrowing cast + * the type needs, if any. + */ + private final case class BooleanNumerals(one: String, zero: String, narrowing: Option[String]) + + /** ONE table for both renderings of a BOOLEAN as a number: the one `coerce` renders, which tests + * the value on every document, and the one [[booleanConstant]] renders, which folds a constant. + * `None` for a type that is not a number. + */ + private def booleanNumerals(to: SQLType): Option[BooleanNumerals] = to match { + case SQLTypes.Numeric | SQLTypes.Int => Some(BooleanNumerals("1", "0", None)) + case SQLTypes.BigInt => Some(BooleanNumerals("1L", "0L", None)) + case SQLTypes.Double => Some(BooleanNumerals("1.0", "0.0", None)) + case SQLTypes.Real => Some(BooleanNumerals("1.0f", "0.0f", None)) + case SQLTypes.SmallInt => Some(BooleanNumerals("1", "0", Some("(short)"))) + case SQLTypes.TinyInt => Some(BooleanNumerals("1", "0", Some("(byte)"))) + case _ => None + } + + /** A boolean CONSTANT as a number of type `to` -- `1L` / `0L` for a BIGINT -- or `None` for a + * type that is not a number. + * + * πŸ”΄ Not `coerce`'s `(true ? 1L : 0L)`: Elasticsearch 6.8's Painless refuses a conditional whose + * condition is a constant ("Extraneous conditional statement"), so a simple `CASE b WHEN '1'` + * (or `WHEN TRUE`) failed to compile there. + */ + private[sql] def booleanConstant(value: Boolean, to: SQLType): Option[String] = + booleanNumerals(to).map { case BooleanNumerals(one, zero, narrowing) => + s"${narrowing.getOrElse("")}${if (value) one else zero}" + } + def canConvert(from: SQLType, to: SQLType): Boolean = { // 1. Identity diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala index c092bdb25..62b65dc65 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/ParserSpec.scala @@ -1175,8 +1175,18 @@ class ParserSpec extends AnyFlatSpec with Matchers { } it should "reject GREATEST with definitively non-numeric args" in { + // once the column types are known -- never when the statement is parsed (the lead's ruling of + // 2026-10-05: no TYPE refusal before the column types are known) val result = Parser("SELECT GREATEST('a', 'b') AS hi FROM products") - result.isLeft shouldBe true + result.isRight shouldBe true + result match { + case Right(single: SingleSearch) => + single + .update(Some(Table("products", columns = Nil))) + .validateResolved() + .isLeft shouldBe true + case other => fail(s"$other") + } } // Story 14.6 β€” case-insensitivity of the Epic-14 keywords (AC 4): every new @@ -1505,7 +1515,7 @@ class ParserSpec extends AnyFlatSpec with Matchers { .map(_.source) .getOrElse("") should include( """def param1 = ctx.birthdate; - |def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); + |def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(" ", "T"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); |def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); |def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); |def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); @@ -1521,16 +1531,16 @@ class ParserSpec extends AnyFlatSpec with Matchers { println(schema.defaultPipeline.ddl) val json = schema.defaultPipeline.json println(json) - json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" + json shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" val indexMappings = schema.indexMappings println(indexMappings) - indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"id":{"data_type":"INT","not_null":"true","comment":"user identifier"},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"birthdate":{"data_type":"DATE","not_null":"false"},"age":{"data_type":"INT","not_null":"false","script":{"sql":"TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5"}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4"}}}}},"type":"regular"}}""".stripMargin + indexMappings.toString shouldBe """{"properties":{"id":{"type":"integer"},"name":{"type":"text","fields":{"raw":{"type":"keyword"}},"analyzer":"french","search_analyzer":"french"},"birthdate":{"type":"date"},"age":{"type":"integer"},"ingested_at":{"type":"date"},"profile":{"type":"object","properties":{"bio":{"type":"text"},"followers":{"type":"integer"},"join_date":{"type":"date"},"seniority":{"type":"integer"}}}},"dynamic":false,"_meta":{"primary_key":["id"],"partition_by":{"column":"birthdate","granularity":"M"},"columns":{"id":{"data_type":"INT","not_null":"true","comment":"user identifier"},"name":{"data_type":"VARCHAR","not_null":"false","default_value":"anonymous","multi_fields":{"raw":{"data_type":"KEYWORD","not_null":"false","comment":"sortable"}}},"birthdate":{"data_type":"DATE","not_null":"false"},"age":{"data_type":"INT","not_null":"false","script":{"sql":"TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)","column":"age","painless":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5"}},"ingested_at":{"data_type":"TIMESTAMP","not_null":"false","default_value":"_ingest.timestamp"},"profile":{"data_type":"STRUCT","not_null":"false","comment":"user profile","multi_fields":{"bio":{"data_type":"VARCHAR","not_null":"false"},"followers":{"data_type":"INT","not_null":"false"},"join_date":{"data_type":"DATE","not_null":"false"},"seniority":{"data_type":"INT","not_null":"false","script":{"sql":"DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)","column":"profile.seniority","painless":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4"}}}}},"type":"regular"}}""".stripMargin val indexSettings = schema.indexSettings println(indexSettings) indexSettings.toString shouldBe """{"index":{}}""" val pipeline = schema.defaultPipelineNode println(pipeline) - pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" + pipeline.toString shouldBe """{"description":"CREATE OR REPLACE PIPELINE users_ddl_default_pipeline WITH PROCESSORS (name SET DEFAULT 'anonymous', age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)), ingested_at SET DEFAULT _ingest.timestamp, profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY)), PARTITION BY birthdate (MONTH), PRIMARY KEY (id))","processors":[{"set":{"description":"name SET DEFAULT 'anonymous'","field":"name","ignore_failure":true,"value":"anonymous","if":"ctx.name == null"}},{"script":{"description":"age INT SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE))","lang":"painless","source":"def param1 = ctx.birthdate; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); def param5 = Long.valueOf((((param4.getYear() * 12L + param4.getMonthValue()) * 4294967296L + (param4.getDayOfMonth() - 1) * 86400000L + param4.getLong(ChronoField.MILLI_OF_DAY)) - ((param2.getYear() * 12L + param2.getMonthValue()) * 4294967296L + (param2.getDayOfMonth() - 1) * 86400000L + param2.getLong(ChronoField.MILLI_OF_DAY))) / 4294967296L / 12); ctx.age = (param1 == null) ? null : param5","ignore_failure":true}},{"set":{"description":"ingested_at SET DEFAULT _ingest.timestamp","field":"ingested_at","ignore_failure":true,"value":"{{_ingest.timestamp}}","if":"ctx.ingested_at == null"}},{"script":{"description":"profile.seniority INT SCRIPT AS (DATE_DIFF(CURRENT_DATE, profile.join_date, DAY))","lang":"painless","source":"def param1 = ctx.profile?.join_date; def param2 = (param1 instanceof String ? (param1.length() > 10 ? ZonedDateTime.parse((param1).replace(\" \", \"T\"), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z')).truncatedTo(ChronoUnit.DAYS) : LocalDate.parse((param1).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\")).atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of('Z')); def param4 = Long.valueOf(ChronoUnit.DAYS.between(LocalDate.from(param2), LocalDate.from(param3))); ctx.profile.seniority = (param1 == null) ? null : param4","ignore_failure":true}},{"date_index_name":{"description":"PARTITION BY birthdate (MONTH)","field":"birthdate","date_rounding":"M","date_formats":["yyyy-MM"],"index_name_prefix":"users-","ignore_failure":true}},{"set":{"description":"PRIMARY KEY (id)","field":"_id","value":"{{id}}","ignore_failure":false,"ignore_empty_value":false}}]}""" // Reconstruct EsIndex val mappings = mapper.createObjectNode() mappings.set("mappings", indexMappings) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala index 46a5420f1..6c73e51e1 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/parser/SetOperatorSpec.scala @@ -236,15 +236,22 @@ class SetOperatorSpec extends AnyFlatSpec with Matchers { } "Branch types" should "be validated where both sides are typeable" in { - rejects("SELECT 1 AS n FROM t UNION SELECT 'a' AS n FROM u", "compatible types at column 1") - rejects( - "SELECT CAST(x AS BIGINT) AS v FROM t UNION ALL SELECT CAST(y AS VARCHAR) AS v FROM u", - "compatible types at column 1" - ) + // A TYPE rule, asked once each branch's schema is attached (`SearchApi.resolveWithSchema`), + // never when the statement is parsed (the lead's ruling of 2026-10-05). + def branchTypes(sql: String): Either[String, Unit] = + MultiSearch.branchTypes(parse(sql).requests) + Seq( + "SELECT 1 AS n FROM t UNION SELECT 'a' AS n FROM u", + "SELECT CAST(x AS BIGINT) AS v FROM t UNION ALL SELECT CAST(y AS VARCHAR) AS v FROM u" + ).foreach { sql => + withClue(s"[$sql] ") { + branchTypes(sql).swap.getOrElse("") should include("compatible types at column 1") + } + } // numeric family - Parser("SELECT 1 AS n FROM t UNION SELECT 2.5 AS n FROM u").isRight shouldBe true - // a bare column is `Any` at parse time β€” the core seam re-checks with schemas attached - Parser("SELECT a FROM t UNION SELECT 'x' FROM u").isRight shouldBe true + branchTypes("SELECT 1 AS n FROM t UNION SELECT 2.5 AS n FROM u") shouldBe Right(()) + // a bare column is `Any` until its schema is attached β€” the core seam checks with schemas + branchTypes("SELECT a FROM t UNION SELECT 'x' FROM u") shouldBe Right(()) } it should "validate a set operation INSIDE a derived table one level down" in { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/perf/ResolutionCostProbe.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/perf/ResolutionCostProbe.scala new file mode 100644 index 000000000..ff566ae8a --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/perf/ResolutionCostProbe.scala @@ -0,0 +1,529 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.perf + +import app.softnetwork.elastic.sql.function.aggregate.BucketScriptAggregation +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.{ + CreateTable, + MetricSelectorScript, + MultiSearch, + SingleSearch +} +import app.softnetwork.elastic.sql.schema.{Column, Table} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} + +import scala.util.{Failure, Success, Try} + +/** Per-statement cost of everything a statement pays before Elasticsearch sees it, STAGE BY STAGE + * -- the companion of [[ParseCostProbe]] (parsing only) and [[EmissionCostProbe]] (parsing and one + * predicate's translation). + * + * Each iteration is one request's worth of work on a FRESH parse, timed per stage: + * + * - `parse` -- `Parser(sql)`, syntax and structure (`validate()`); + * - `attach` -- `updateAll` with the corpus schemas, what `SearchApi.resolveWithSchema` does + * once the mapping is read; + * - `validate` -- `validateResolved()`, the rules that need the column types (for a set + * operation, every branch's, then the branch type check); + * - `emit` -- the Painless the bridge renders from the statement: every script field, the WHERE + * criteria (query context), every per-group calculation (`bucket_script`) and the HAVING + * selector; + * + * and the `total` of the four. A statement refused at a stage stops there, as a request does, and + * says so in its verdict; a DDL statement is parsed, and its computed columns' scripts read. + * + * Corpus: [[ParseCostProbe]] 's statements, [[EmissionCostProbe]] 's, the TYPE rules that are + * asked once the column types are known (comparisons, IN, BETWEEN, NULLIF, CASE, GREATEST / LEAST, + * intervals, arithmetic, set operations -- their true mismatches and the statements they used to + * refuse although well typed), and the date-arithmetic request bodies. Then, as one block, the + * date-arithmetic POPULATION: every operand against every operand for `+ - * / %`, a few + * iterations each, reported as the sum of the per-statement medians per stage. + * + * Same protocol as the other probes: a global warm-up, a per-statement discarded warm-up, then + * `runs` timed iterations, reported as medians in microseconds. It uses only what the base already + * has, so the SAME file runs in a control tree. + * + * {{{ + * sbt "sql/Test/runMain app.softnetwork.elastic.sql.perf.ResolutionCostProbe" + * sbt "sql/Test/runMain app.softnetwork.elastic.sql.perf.ResolutionCostProbe 100 300 out.tsv" + * }}} + * + * With an output path, the per-statement table is written there as TSV and the population block + * beside it (`.population.tsv`), so two trees can be compared mechanically. + * + * ==This probe NEVER asserts== + * It prints and exits (issues #269/#270: a timing assertion is a CI-flake liability). Compare two + * trees by running it in each, INTERLEAVED, in one session -- an absolute number from another + * session is noise. + */ +object ResolutionCostProbe { + + val DefaultWarmup: Int = 100 + val DefaultRuns: Int = 300 + val PopulationWarmup: Int = 2 + val PopulationRuns: Int = 5 + + private def table(name: String, columns: (String, SQLType)*): Table = + Table(name, columns = columns.map { case (n, t) => Column(n, t) }.toList) + + private val schemas: Map[String, Table] = Map( + "t" -> table( + "t", + "id" -> SQLTypes.Keyword, + "g" -> SQLTypes.Keyword, + "s" -> SQLTypes.Keyword, + "name" -> SQLTypes.Keyword, + "other" -> SQLTypes.Keyword, + "num" -> SQLTypes.Keyword, + "n" -> SQLTypes.Int, + "m" -> SQLTypes.Int, + "x" -> SQLTypes.Double, + "amount" -> SQLTypes.Double, + "d" -> SQLTypes.Date, + "d2" -> SQLTypes.Date, + "ts" -> SQLTypes.Timestamp, + "ts2" -> SQLTypes.Timestamp, + "b" -> SQLTypes.Boolean + ), + "orders" -> table( + "orders", + "id" -> SQLTypes.Keyword, + "name" -> SQLTypes.Keyword, + "category" -> SQLTypes.Keyword, + "amount" -> SQLTypes.Double, + "created_at" -> SQLTypes.Timestamp, + "status" -> SQLTypes.Keyword, + "country" -> SQLTypes.Keyword, + "city" -> SQLTypes.Keyword, + "label" -> SQLTypes.Keyword, + "customer_id" -> SQLTypes.Keyword + ), + "customers" -> table( + "customers", + "id" -> SQLTypes.Keyword, + "cid" -> SQLTypes.Int, + "region" -> SQLTypes.Keyword, + "name" -> SQLTypes.Keyword + ), + "emp" -> table( + "emp", + "name" -> SQLTypes.Keyword, + "salary" -> SQLTypes.Double, + "department" -> SQLTypes.Keyword + ) + ) + + /** [[EmissionCostProbe]]'s predicate shapes (its list is private to it). */ + private val EmissionStatements: List[(String, String)] = List( + "plain column, numeric" -> "SELECT id FROM t WHERE n > 1", + "plain column, keyword" -> "SELECT id FROM t WHERE name = 'x'", + "plain column, temporal" -> "SELECT id FROM t WHERE d = CAST('2025-01-31' AS DATE)", + "folded function (YEAR)" -> "SELECT id FROM t WHERE YEAR(d) = 2025", + "folded function (DATE_TRUNC)" -> + "SELECT id FROM t WHERE DATE_TRUNC(d, MONTH) = CAST('2025-01-01' AS DATE)", + "expression function (WEEKDAY)" -> "SELECT id FROM t WHERE WEEKDAY(d) = 0", + "expression function (DATE_FORMAT)" -> "SELECT id FROM t WHERE DATE_FORMAT(d, 'yyyy') = '2025'", + "expression function (LAST_DAY)" -> + "SELECT id FROM t WHERE LAST_DAY(d) = CAST('2025-01-31' AS DATE)", + "conversion (CAST)" -> "SELECT id FROM t WHERE CAST(num AS INT) = 9", + "conversion (TRY_CAST)" -> "SELECT id FROM t WHERE TRY_CAST(num AS INT) = 9", + "conditional (ISNULL)" -> "SELECT id FROM t WHERE ISNULL(name) = true", + "conditional (NULLIF, cast)" -> "SELECT id FROM t WHERE NULLIF(CAST(num AS BIGINT), 0) > 1", + "conditional (NULLIF, function)" -> "SELECT id FROM t WHERE NULLIF(UPPER(name), 'X') = 'A'", + "string function (UPPER)" -> "SELECT id FROM t WHERE UPPER(name) = 'A'", + "math function (ABS)" -> "SELECT id FROM t WHERE ABS(amount) > 10", + "division, two integer columns" -> "SELECT id FROM t WHERE n / m > 1", + "division, literal divisor" -> "SELECT id FROM t WHERE amount / 2 > 1", + "IN, numeric" -> "SELECT id FROM t WHERE CAST(num AS BIGINT) IN (9, 12)", + "IN, strings" -> "SELECT id FROM t WHERE UPPER(name) IN ('A', 'B')", + "both sides" -> "SELECT id FROM t WHERE YEAR(d) = YEAR(ts)", + "composite AND/OR" -> + "SELECT id FROM t WHERE WEEKDAY(d) = 0 AND UPPER(name) = 'A' OR ABS(amount) > 10" + ) + + /** The TYPE rules asked once the column types are known: one true mismatch per rule, then the + * statements that are well typed -- the ones parse-time rules used to refuse, a literal against + * a bare column, functions over dates. + */ + private val TypeRuleStatements: List[(String, String)] = List( + "comparison, literal mismatch" -> "SELECT id FROM t WHERE n + 1 > 'abc'", + "comparison, column mismatch" -> "SELECT id FROM t WHERE s = n", + "comparison, TIME" -> "SELECT id FROM t WHERE CAST(ts AS TIME) > ts", + "IN, mismatch" -> "SELECT id FROM t WHERE CAST(n AS INT) IN ('a', 'b')", + "IN (HAVING), mismatch" -> "SELECT g FROM t GROUP BY g HAVING COUNT(n) IN ('a', 'b')", + "BETWEEN bounds, mismatch" -> "SELECT id FROM t WHERE n BETWEEN 1 AND 'a'", + "BETWEEN operand, mismatch" -> "SELECT id FROM t WHERE CAST(s AS INT) BETWEEN 'a' AND 'b'", + "NULLIF, mismatch" -> "SELECT NULLIF(s, 1) AS v FROM t", + "NULLIF, kinds" -> "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP)) AS v FROM t", + "CASE WHEN, non-boolean" -> "SELECT CASE WHEN n THEN 1 ELSE 0 END AS v FROM t", + "CASE , mismatch" -> "SELECT CASE n WHEN 'a' THEN 1 ELSE 0 END AS v FROM t", + "GREATEST, strings" -> "SELECT GREATEST('a', 'b') AS v FROM t", + "LEAST, dates" -> "SELECT LEAST(d, d2) AS v FROM t", + "date arithmetic, clock" -> "SELECT CURRENT_DATE * 2 AS v FROM t", + "date arithmetic, columns" -> "SELECT d + d2 AS v FROM t", + "arithmetic, column mismatch" -> "SELECT n + s AS v FROM t", + "arithmetic, literal mismatch" -> "SELECT 1 + 'abc' AS v FROM t", + "set operation, mismatch" -> "SELECT 1 AS n FROM t UNION ALL SELECT 'a' AS n FROM t", + "set operation, agreeing" -> "SELECT id FROM t UNION ALL SELECT g FROM t", + "well typed, CURRENT_DATE - d" -> "SELECT id FROM t WHERE CURRENT_DATE - d > 30", + "well typed, d + 7" -> "SELECT id FROM t WHERE d + 7 > CURRENT_DATE", + "well typed, NOW() - ts" -> "SELECT id FROM t WHERE NOW() - ts > 1", + "well typed, CAST - d" -> "SELECT id FROM t WHERE CAST(ts AS DATE) - d = 0", + "well typed, d + 1 = DATE" -> "SELECT id FROM t WHERE d + 1 = CAST('2024-02-01' AS DATE)", + "well typed, d + 1 > literal" -> "SELECT id FROM t WHERE d + 1 > '2024-01-01'", + "well typed, HAVING alias" -> + "SELECT g, MAX(d) + 1 AS v FROM t GROUP BY g HAVING v > CAST('2024-01-01' AS DATE)", + "well typed, CASE WHEN b" -> "SELECT CASE WHEN b THEN 1 ELSE 0 END AS v FROM t", + "literal vs column, =" -> "SELECT id FROM t WHERE n = '5'", + "literal vs column, date" -> "SELECT id FROM t WHERE d >= '2024-01-01'", + "literal vs column, IN" -> "SELECT id FROM t WHERE n IN ('1', '2')", + "literal vs column, BETWEEN" -> "SELECT id FROM t WHERE n BETWEEN '1' AND '9'", + "interval over a date" -> "SELECT id FROM t WHERE DATE_ADD(d, INTERVAL 1 DAY) > ts", + "COALESCE against the clock" -> "SELECT id FROM t WHERE COALESCE(d, d2) > CURRENT_DATE", + "CASE, many branches" -> + ("SELECT CASE WHEN n > 10 THEN 'a' WHEN n > 5 THEN 'b' WHEN b THEN 'c' ELSE 'd' END AS v " + + "FROM t WHERE x > 1 AND s IN ('p', 'q') AND d BETWEEN '2024-01-01' AND '2024-12-31'") + ) + + /** The request bodies date arithmetic renders: rows, comparisons against an instant, the clock in + * a per-group calculation, a cast over arithmetic, an aggregate's own type, a DDL computed + * column. The typed NULLs come last: on a tree where `CAST(NULL AS ..)` retypes the shared NULL + * literal, nothing after them can be affected. + */ + private val DateArithmeticStatements: List[(String, String)] = List( + "row, d + 1" -> "SELECT d + 1 AS v FROM t", + "row, ts - 1.5" -> "SELECT ts - 1.5 AS v FROM t", + "row, d - d2" -> "SELECT d - d2 AS v FROM t", + "row, ts - ts2" -> "SELECT ts - ts2 AS v FROM t", + "row, d + '1'" -> "SELECT d + '1' AS v FROM t", + "row, COALESCE(d, d2) - d" -> "SELECT COALESCE(d, d2) - d AS v FROM t", + "row, NULLIF(d, d2) + 1" -> "SELECT NULLIF(d, d2) + 1 AS v FROM t", + "row, CASE - d" -> "SELECT CASE WHEN n > 0 THEN d ELSE d2 END - d AS v FROM t", + "where, d + 1 > ts" -> "SELECT id FROM t WHERE d + 1 > ts", + "where, ts > d + 1" -> "SELECT id FROM t WHERE ts > d + 1", + "where, CAST(d) >= ts" -> "SELECT id FROM t WHERE CAST(d AS TIMESTAMP) >= ts", + "where, LAST_DAY(d) = ts" -> "SELECT id FROM t WHERE LAST_DAY(d) = ts", + "where, d - d2 > 3" -> "SELECT id FROM t WHERE d - d2 > 3", + "group, CURRENT_DATE - MAX(d)" -> "SELECT g, CURRENT_DATE - MAX(d) AS v FROM t GROUP BY g", + "group, NOW() - MAX(ts)" -> "SELECT g, NOW() - MAX(ts) AS v FROM t GROUP BY g", + "group, DATEDIFF(clock, MAX)" -> "SELECT g, DATEDIFF(CURRENT_DATE, MAX(d)) AS v FROM t GROUP BY g", + "group, MAX(d) - MIN(d)" -> "SELECT g, MAX(d) - MIN(d) AS v FROM t GROUP BY g", + "group, COUNT(d) - 1" -> "SELECT g, COUNT(d) - 1 AS v FROM t GROUP BY g", + "group, MAX(d) - AVG(n)" -> "SELECT g, MAX(d) - AVG(n) AS v FROM t GROUP BY g", + "having, alias of MAX - MIN" -> "SELECT g, MAX(d) - MIN(d) AS k FROM t GROUP BY g HAVING k > 1", + "cast, (n + x)::BIGINT" -> "SELECT (n + x)::BIGINT AS v FROM t", + "cast, (d + 1)::TIMESTAMP" -> "SELECT (d + 1)::TIMESTAMP AS v FROM t", + "cast, (x * 2)::INT" -> "SELECT (x * 2)::INT AS v FROM t", + "DDL, computed date columns" -> + ("CREATE TABLE ev2 (id INT, d DATE, d2 DATE, gap INT SCRIPT AS (d - d2), " + + "next DATE SCRIPT AS (COALESCE(d, d2) + 1), PRIMARY KEY (id))"), + "typed NULL, CAST(NULL AS DATE) - d" -> "SELECT CAST(NULL AS DATE) - d AS v FROM t", + "typed NULL, d + CAST(NULL AS INT)" -> "SELECT d + CAST(NULL AS INT) AS v FROM t" + ) + + val Statements: List[(String, String, String)] = + ParseCostProbe.Statements.map { case (l, s) => ("parse", l, s) } ++ + EmissionStatements.map { case (l, s) => ("emission", l, s) } ++ + TypeRuleStatements.map { case (l, s) => ("type rules", l, s) } ++ + DateArithmeticStatements.map { case (l, s) => ("date arithmetic", l, s) } + + /** The date-arithmetic population: every operand on both sides of every operator. */ + private val PopulationOperands: List[(String, Boolean)] = List( + "d" -> false, + "ts" -> false, + "MAX(d)" -> true, + "MAX(ts)" -> true, + "2" -> false, + "1.5" -> false, + "n" -> false, + "x" -> false, + "CURRENT_DATE" -> false, + "NOW()" -> false, + "NULL" -> false, + "CASE WHEN n > 0 THEN d ELSE d2 END" -> false, + "COALESCE(d, d2)" -> false, + "COALESCE(ts, d)" -> false, + "NULLIF(d, d2)" -> false, + "GREATEST(n, 2)" -> false, + "LEAST(x, 1.5)" -> false, + "AVG(n)" -> true, + "SUM(n)" -> true, + "COUNT(d)" -> true, + "'1'" -> false, + "'1.5'" -> false, + "'x'" -> false, + // typed NULLs last, for the reason given on `DateArithmeticStatements` + "CAST(NULL AS DATE)" -> false, + "CAST(NULL AS INT)" -> false + ) + + val Population: List[String] = + for { + (l, la) <- PopulationOperands + (r, ra) <- PopulationOperands + op <- List("+", "-", "*", "/", "%") + } yield + if (la || ra) s"SELECT g, $l $op $r AS x FROM t GROUP BY g" + else s"SELECT $l $op $r AS x FROM t" + + private def queryContext(): PainlessContext = PainlessContext(context = PainlessContextType.Query) + + /** What the bridge renders from a resolved statement, as Painless text. */ + private def emission(s: SingleSearch): Int = { + var n = 0 + s.scriptFields.filterNot(_.isAggregation).foreach { f => + val ctx = queryContext() + val body = f.painless(Some(ctx)) + n += (ctx.toString + body).length + } + s.where.flatMap(_.criteria).foreach { c => + val ctx = queryContext() + val body = c.painless(Some(ctx)) + n += (ctx.toString + body).length + } + s.select.fields.filter(_.isBucketScript).foreach { f => + n += BucketScriptAggregation(f.identifier).update(s).identifier.painless(None).length + } + s.having.flatMap(_.criteria).foreach(c => n += MetricSelectorScript.metricSelector(c).length) + n + } + + /** One timed iteration: nanoseconds per stage (`-1` = not reached) and the verdict. */ + private final case class Iteration( + parse: Long, + attach: Long, + validate: Long, + emit: Long, + verdict: String, + sink: Int + ) { + def total: Long = parse + math.max(attach, 0L) + math.max(validate, 0L) + math.max(emit, 0L) + } + + private def iteration(sql: String): Iteration = { + val t0 = System.nanoTime() + val parsed = Parser(sql) + val t1 = System.nanoTime() + parsed match { + case Left(_) => Iteration(t1 - t0, -1, -1, -1, "parse-refused", 0) + case Right(single: SingleSearch) => + val attached = Try(single.updateAll(schemas)) + val t2 = System.nanoTime() + attached match { + case Failure(e) => Iteration(t1 - t0, t2 - t1, -1, -1, s"attach-throws", e.hashCode()) + case Success(resolved) => + val verdict = Try(resolved.validateResolved()) + val t3 = System.nanoTime() + verdict match { + case Success(Right(())) => + val emitted = Try(emission(resolved)) + val t4 = System.nanoTime() + Iteration( + t1 - t0, + t2 - t1, + t3 - t2, + t4 - t3, + if (emitted.isSuccess) "ok" else "emit-throws", + emitted.getOrElse(0) + ) + case Success(Left(_)) => + Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "resolve-refused", 0) + case Failure(_) => Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "validate-throws", 0) + } + } + case Right(multi: MultiSearch) => + val attached = Try(multi.requests.map(_.updateAll(schemas))) + val t2 = System.nanoTime() + attached match { + case Failure(e) => Iteration(t1 - t0, t2 - t1, -1, -1, "attach-throws", e.hashCode()) + case Success(branches) => + val verdict = Try( + branches + .map(_.validateResolved()) + .collectFirst { case l @ Left(_) => l } + .getOrElse(MultiSearch.branchTypes(branches)) + ) + val t3 = System.nanoTime() + verdict match { + case Success(Right(())) => + val emitted = Try(branches.map(emission).sum) + val t4 = System.nanoTime() + Iteration( + t1 - t0, + t2 - t1, + t3 - t2, + t4 - t3, + if (emitted.isSuccess) "ok" else "emit-throws", + emitted.getOrElse(0) + ) + case Success(Left(_)) => + Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "resolve-refused", 0) + case Failure(_) => Iteration(t1 - t0, t2 - t1, t3 - t2, -1, "validate-throws", 0) + } + } + case Right(ddl: CreateTable) => + val scripts = Try(ddl.schema.columns.flatMap(_.script).map(_.source.length).sum) + val t2 = System.nanoTime() + Iteration(t1 - t0, -1, -1, t2 - t1, "ddl", scripts.getOrElse(0)) + case Right(other) => Iteration(t1 - t0, -1, -1, -1, "other", other.hashCode()) + } + } + + private def median(values: Seq[Long]): Long = + if (values.isEmpty || values.forall(_ < 0)) -1L + else { + val sorted = values.filter(_ >= 0).sorted.toArray + val n = sorted.length + if (n % 2 == 1) sorted(n / 2) else (sorted(n / 2 - 1) + sorted(n / 2)) / 2 + } + + private def p95(values: Seq[Long]): Long = { + val sorted = values.sorted.toArray + sorted(math.min(sorted.length - 1, (sorted.length.toLong * 95L / 100L).toInt)) + } + + /** Locale-independent: this machine's default locale renders a decimal comma. */ + private def micros(nanos: Long): String = + if (nanos < 0) "-" + else String.format(java.util.Locale.ROOT, "%.1f", java.lang.Double.valueOf(nanos / 1000.0)) + + private final case class Row( + group: String, + label: String, + verdict: String, + parse: Long, + attach: Long, + validate: Long, + emit: Long, + total: Long, + totalP95: Long + ) + + private def measure( + group: String, + label: String, + sql: String, + warmup: Int, + runs: Int + ): (Row, Int) = { + var sink = 0 + for (_ <- 1 to warmup) sink ^= iteration(sql).sink + val samples = (1 to runs).map(_ => iteration(sql)) + samples.foreach(s => sink ^= s.sink) + val totals = samples.map(_.total) + ( + Row( + group, + label, + samples.head.verdict, + median(samples.map(_.parse)), + median(samples.map(_.attach)), + median(samples.map(_.validate)), + median(samples.map(_.emit)), + median(totals), + p95(totals) + ), + sink + ) + } + + def main(args: Array[String]): Unit = { + val warmup = if (args.length > 0) args(0).toInt else DefaultWarmup + val runs = if (args.length > 1) args(1).toInt else DefaultRuns + val out = if (args.length > 2) Some(args(2)) else None + if (warmup < 0 || runs < 1) { + throw new RuntimeException( + s"ResolutionCostProbe: warmup must be >= 0 and runs >= 1 (got warmup=$warmup runs=$runs)" + ) + } + + // Global warm-up: every object initialised and the hot paths compiled before any timing. + var sink = 0 + for (_ <- 1 to warmup; (_, _, sql) <- Statements) sink ^= iteration(sql).sink + + val rows = Statements.map { case (group, label, sql) => + val (row, s) = measure(group, label, sql, warmup, runs) + sink ^= s + row + } + + val header = + f"${"group"}%-16s ${"statement"}%-36s ${"verdict"}%-16s ${"parse"}%8s ${"attach"}%8s ${"validate"}%9s ${"emit"}%8s ${"total"}%9s ${"p95"}%9s" + println() + println(s"ResolutionCostProbe - warmup=$warmup runs=$runs, medians in microseconds") + println(header) + println("-" * header.length) + rows.foreach { r => + println( + f"${r.group}%-16s ${r.label.take(36)}%-36s ${r.verdict}%-16s ${micros(r.parse)}%8s ${micros( + r.attach + )}%8s ${micros(r.validate)}%9s ${micros(r.emit)}%8s ${micros(r.total)}%9s ${micros(r.totalP95)}%9s" + ) + } + println("-" * header.length) + def sum(f: Row => Long): Long = rows.map(f).filter(_ >= 0).sum + println( + f"${"SUM of medians"}%-70s ${micros(sum(_.parse))}%8s ${micros(sum(_.attach))}%8s ${micros( + sum(_.validate) + )}%9s ${micros(sum(_.emit))}%8s ${micros(sum(_.total))}%9s" + ) + + // The population block: a few iterations per statement, summed per stage. + val population = Population.map { sql => + val (row, s) = measure("population", sql, sql, PopulationWarmup, PopulationRuns) + sink ^= s + row + } + def popSum(f: Row => Long): Long = population.map(f).filter(_ >= 0).sum + val verdicts = + population.groupBy(_.verdict).map { case (v, rs) => s"$v=${rs.size}" }.toSeq.sorted + println() + println( + s"population: ${population.size} statements (${verdicts.mkString(", ")}), " + + s"$PopulationRuns runs each, SUM of medians in microseconds: parse ${micros(popSum(_.parse))}, " + + s"attach ${micros(popSum(_.attach))}, validate ${micros(popSum(_.validate))}, " + + s"emit ${micros(popSum(_.emit))}, total ${micros(popSum(_.total))}" + ) + println() + + out.foreach { path => + def write(file: String, lines: Seq[Row]): Unit = { + val sb = new StringBuilder + sb.append( + "group\tstatement\tverdict\tparse_ns\tattach_ns\tvalidate_ns\temit_ns\ttotal_ns\ttotal_p95_ns\n" + ) + lines.foreach { r => + sb.append( + s"${r.group}\t${r.label}\t${r.verdict}\t${r.parse}\t${r.attach}\t${r.validate}\t${r.emit}\t${r.total}\t${r.totalP95}\n" + ) + } + java.nio.file.Files.write( + java.nio.file.Paths.get(file), + sb.toString.getBytes(java.nio.charset.StandardCharsets.UTF_8) + ) + } + write(path, rows) + write(s"$path.population.tsv", population) + println(s"ResolutionCostProbe: wrote $path and $path.population.tsv") + } + + // Keep `sink` observable so the JIT cannot eliminate the work as dead code. + if (sink == Int.MinValue) println("unreachable sink guard") + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala new file mode 100644 index 000000000..ac1559c00 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/DateArithmeticTypeSpec.scala @@ -0,0 +1,435 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.operator.math.ArithmeticExpression +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.{SQLType, SQLTypes} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** The date-arithmetic rules, as the TYPE every `+ - * / %` expression with a temporal operand + * gets, or the refusal by name it earns -- asked of a generated population once its schema is + * attached, and NEVER when the statement is parsed: no type is refused before the column types are + * known (the lead's ruling of 2026-10-05). + * + * The expectation is written HERE, from the ruling, and never read from the implementation: + * + * | Expression | Type | + * |:-------------------------------------------------------------------|:--------------------------------| + * | DATE - DATE | BIGINT | + * | TIMESTAMP - TIMESTAMP, a DATE and a TIMESTAMP | DOUBLE | + * | DATE +/- a whole number | DATE | + * | DATE +/- a fractional number | TIMESTAMP | + * | TIMESTAMP +/- a number | TIMESTAMP | + * | a number + a temporal | as the temporal + it | + * | a temporal and NULL (+ or -) | not refused (the value is NULL) | + * | temporal + temporal, a temporal under * / %, a number - a temporal | refused by name | + * | a temporal and anything that is neither a number nor a temporal | refused by name | + * + * What an operand IS for these rules: + * - `NOW()` is a DATETIME, which these rules read as an instant, like a TIMESTAMP; + * - an aggregate is the metric it produces: `COUNT` a BIGINT, `AVG` a DOUBLE, `SUM` its column's + * number, `MIN` / `MAX` what they aggregate; + * - `CASE`, `COALESCE`, `GREATEST` and `LEAST` are the least common supertype of their branches + * or arguments, `NULLIF` its first argument -- `GREATEST` / `LEAST` over dates included, a + * DATE when every argument is one, a TIMESTAMP when they mix; + * - a typed NULL is its type (`CAST(NULL AS DATE)` is a DATE), a bare `NULL` has none; + * - a string literal that is a number is that number (`'1'`, `'1.5'`), so a number of days + * beside a temporal -- elasticsql's OWN rule, not PostgreSQL's: PostgreSQL 16 refuses `d + + * '1'` (`operator is not unique: date + unknown`), reads the `'1'` of `d - '1'` as a date + * (`invalid input syntax for type date`) and reads `ts + '1'` as one second; one that spells a + * date or a timestamp is, beside a temporal, that temporal read as the other operand's type + * (`d - '2024-01-31'` is DATE minus DATE, as PostgreSQL reads it) and a string beside anything + * else; any other is a string; + * - a `date` field elasticsql never declared (`u`, an index created by a bulk load) holds + * instants: a TIMESTAMP, typed so at the declaration seam -- DESCRIBE shows it so too. + */ +class DateArithmeticTypeSpec extends AnyFlatSpec with Matchers { + + private val schema: SchemaTable = SchemaTable( + "ev", + columns = List( + Column("g", SQLTypes.Keyword), + Column("d", SQLTypes.Date), + Column("d2", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("n", SQLTypes.Int), + Column("x", SQLTypes.Double), + // an Elasticsearch `date` with no elasticsql declaration (no `_meta` data type), typed at + // the declaration seam (`schema.IndexField`) + Index("raw", """{"mappings":{"properties":{"u":{"type":"date"}}},"settings":{}}""").schema + .find("u") + .getOrElse(fail("no undeclared column u")) + ) + ) + + private val ddlColumns = "g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, n INT, x DOUBLE" + + /** What the ruling sees in an operand. */ + private sealed trait Kind + private case object DateKind extends Kind + private case object InstantKind extends Kind + private case object Whole extends Kind + private case object Fractional extends Kind + private case object NullKind extends Kind + private case object TextKind extends Kind + + /** A string literal that spells a date or a timestamp: what it IS depends on the other operand. + */ + private case object TemporalText extends Kind + + /** One operand of the population: its SQL, what the ruling sees in it, the type name a refusal + * names it by, whether it is an aggregate, and whether a computed column can hold it (the DDL + * venue declares every column, so it has no undeclared one, and no aggregate). + */ + private final case class Operand( + sql: String, + kind: Kind, + typeName: String, + aggregate: Boolean = false, + ddl: Boolean = true + ) + + private val operands: Seq[Operand] = Seq( + Operand("d", DateKind, "DATE"), + Operand("ts", InstantKind, "TIMESTAMP"), + Operand("MAX(d)", DateKind, "DATE", aggregate = true, ddl = false), + Operand("MAX(ts)", InstantKind, "TIMESTAMP", aggregate = true, ddl = false), + Operand("2", Whole, "BIGINT"), + Operand("1.5", Fractional, "DOUBLE"), + Operand("n", Whole, "INT"), + Operand("x", Fractional, "DOUBLE"), + Operand("CURRENT_DATE", DateKind, "DATE"), + Operand("NOW()", InstantKind, "DATETIME"), + Operand("NULL", NullKind, "NULL"), + // ---- the fix round's extension (E1) ---- + Operand("u", InstantKind, "TIMESTAMP", ddl = false), + Operand("MAX(u)", InstantKind, "TIMESTAMP", aggregate = true, ddl = false), + Operand("CASE WHEN n > 0 THEN d ELSE d2 END", DateKind, "DATE"), + Operand("COALESCE(d, d2)", DateKind, "DATE"), + Operand("COALESCE(ts, d)", InstantKind, "TIMESTAMP"), + Operand("NULLIF(d, d2)", DateKind, "DATE"), + Operand("GREATEST(n, 2)", Whole, "BIGINT"), + Operand("LEAST(x, 1.5)", Fractional, "DOUBLE"), + Operand("AVG(n)", Fractional, "DOUBLE", aggregate = true, ddl = false), + Operand("SUM(n)", Whole, "INT", aggregate = true, ddl = false), + Operand("COUNT(d)", Whole, "BIGINT", aggregate = true, ddl = false), + Operand("CAST(NULL AS DATE)", DateKind, "DATE"), + Operand("CAST(NULL AS INT)", Whole, "INT"), + Operand("'1'", Whole, "VARCHAR"), + Operand("'1.5'", Fractional, "VARCHAR"), + Operand("'x'", TextKind, "VARCHAR"), + // ---- fix round 2 (G1): GREATEST / LEAST over dates and timestamps ---- + Operand("GREATEST(d, d2)", DateKind, "DATE"), + Operand("LEAST(ts, d)", InstantKind, "TIMESTAMP"), + Operand("GREATEST(MAX(d), MAX(d2))", DateKind, "DATE", aggregate = true, ddl = false), + // ---- fix round 3 (T1): a string literal that spells a date or a timestamp ---- + Operand("'2024-01-31'", TemporalText, "VARCHAR"), + Operand("'2024-01-31T10:00:00Z'", TemporalText, "VARCHAR") + ) + + private val operators: Seq[String] = Seq("+", "-", "*", "/", "%") + + private sealed trait Expected + private final case class Typed(t: SQLType) extends Expected + private case object NullValued extends Expected + private case object Refused extends Expected + private case object NotDateArithmetic extends Expected + + private def temporal(k: Kind): Boolean = k == DateKind || k == InstantKind + private def number(k: Kind): Boolean = k == Whole || k == Fractional + + private def shifted(t: Kind, n: Kind): Typed = + Typed(if (t == DateKind && n == Whole) SQLTypes.Date else SQLTypes.Timestamp) + + /** What an operand IS beside the other: a literal that spells a temporal is, beside a temporal, + * that temporal read as the other's type -- the comparison rule's reading, as PostgreSQL reads + * an untyped literal -- and a string beside anything else. + */ + private def beside(k: Kind, other: Kind): Kind = + if (k != TemporalText) k else if (temporal(other)) other else TextKind + + /** The ruling, restated. */ + private def expected(op: String, left: Kind, right: Kind): Expected = { + val l = beside(left, right) + val r = beside(right, left) + if (!temporal(l) && !temporal(r)) NotDateArithmetic + else + op match { + case "*" | "/" | "%" => Refused + case "+" => + (l, r) match { + case (t, n) if temporal(t) && number(n) => shifted(t, n) + case (n, t) if number(n) && temporal(t) => shifted(t, n) + case (NullKind, _) | (_, NullKind) => NullValued + case _ => Refused + } + case "-" => + (l, r) match { + case (DateKind, DateKind) => Typed(SQLTypes.BigInt) + case (a, b) if temporal(a) && temporal(b) => Typed(SQLTypes.Double) + case (t, n) if temporal(t) && number(n) => shifted(t, n) + case (NullKind, _) | (_, NullKind) => NullValued + case _ => Refused + } + } + } + + private final case class Case(op: String, l: Operand, r: Operand) { + val expression: String = s"${l.sql} $op ${r.sql}" + val outcome: Expected = expected(op, l.kind, r.kind) + val aggregate: Boolean = l.aggregate || r.aggregate + def sql: String = + if (aggregate) s"SELECT g, $expression AS x FROM ev GROUP BY g" + else s"SELECT $expression AS x FROM ev" + } + + private val population: Seq[Case] = + for { + op <- operators + l <- operands + r <- operands + } yield Case(op, l, r) + + /** The refusal names the operator, both operand types and the remedy, and nothing internal. */ + private def namesTheRefusal(c: Case, message: String): Boolean = + message.contains(s"operator ${c.op} ") && + message.contains(s"${c.l.typeName} and ${c.r.typeName}") && + message.contains("subtract two dates for the days between them") && + message.contains("add or subtract a number of days") && + !message.matches("(?s).*(? Some(s"refused at parse (${err.msg}): no TYPE is refused at parse") + case Right(ss: SingleSearch) => + val resolved = ss.update(Some(schema)) + val item = resolved.select.fields + .find(_.fieldAlias.exists(_.alias == "x")) + .getOrElse(fail(s"[${c.sql}] no x item")) + (c.outcome, resolved.validateResolved()) match { + case (Refused, Left(message)) => + if (namesTheRefusal(c, message)) None else Some(s"refused as [$message]") + case (Refused, Right(_)) => + Some(s"accepted, reported ${item.identifier.reportedType}, expected a refusal") + case (Typed(t), Right(_)) => + val reported = item.identifier.reportedType + if (reported == t) None else Some(s"reported $reported, expected $t") + case (NullValued, Right(_)) => None + case (NotDateArithmetic, Right(_)) => + if (ArithmeticExpression.typeErrorsOf(item.identifier).isEmpty) None + else Some("a date-arithmetic refusal for an expression with no temporal") + // a type rule other than the date-arithmetic one (`'x' + 2`), asked once types are known + case (NotDateArithmetic, Left(message)) if !dateArithmeticRefusal(message) => None + case (other, Left(message)) => Some(s"refused as [$message], expected $other") + } + case Right(other) => Some(s"parsed as ${other.getClass.getSimpleName}") + } + + "every + - * / % with a DATE, TIMESTAMP or DATETIME operand" should + "get the ruling's type or be refused by name, once its schema is attached -- never at parse" in { + // Non-vacuity, over the material: every kind on both sides of every operator, and every + // outcome of the ruling present. + population.size shouldBe operators.size * operands.size * operands.size + val outcomes = population.map(_.outcome) + outcomes.count(_ == Refused) should be > 100 + outcomes.collect { case Typed(t) => t }.toSet shouldBe + Set(SQLTypes.BigInt, SQLTypes.Double, SQLTypes.Date, SQLTypes.Timestamp) + outcomes should contain(NullValued) + outcomes should contain(NotDateArithmetic) + + val mismatches = population.flatMap(c => mismatch(c).map(m => s"[${c.expression}] $m")) + info( + s"date arithmetic, query venue: ${population.size} expressions " + + s"(${outcomes.count(_ == Refused)} refused, ${outcomes.count(_.isInstanceOf[Typed])} typed, " + + s"${outcomes.count(_ == NullValued)} NULL, ${outcomes.count(_ == NotDateArithmetic)} " + + s"without a temporal); mismatches: ${mismatches.size}" + ) + withClue(mismatches.take(30).mkString("\n", "\n", "\n")) { + mismatches shouldBe empty + } + } + + /** The DDL venue, as `CREATE TABLE` runs it: `Parser` -> `CreateTable.validate()` -> + * `validateScriptReferences` on the RESOLVED table -- its column types are declared in the + * statement itself. A computed column cannot hold an aggregate, so the population is the + * row-level operands, and every column is declared. + */ + private def ddlVerdict(expression: String, dataType: String): Either[String, String] = + Parser(s"CREATE TABLE ev ($ddlColumns, c $dataType SCRIPT AS ($expression))") match { + case Right(ct: CreateTable) => + Right( + ct.schema.columns + .find(_.name == "c") + .flatMap(_.script) + .map(_.source) + .getOrElse(fail(s"no script processor for [$expression]")) + ) + case Right(other) => fail(s"[$expression] expected a CreateTable, got $other") + case Left(err) => Left(err.msg) + } + + "a computed column over date arithmetic" should + "be created with the ruling's type, or refused by name" in { + val ddlOperands = operands.filter(_.ddl) + val ddlPopulation = population.filter(c => c.l.ddl && c.r.ddl) + ddlPopulation.size shouldBe operators.size * ddlOperands.size * ddlOperands.size + val mismatches = ddlPopulation.flatMap { c => + val dataType = c.outcome match { + case Typed(t) => t.typeId + case _ => "BIGINT" + } + val verdict = ddlVerdict(c.expression, dataType) + ((c.outcome, verdict) match { + case (Refused, Left(message)) => + if (message.startsWith("Column 'c': ") && namesTheRefusal(c, message)) None + else Some(s"refused as [$message]") + case (Refused, Right(source)) => Some(s"created [$source], expected a refusal") + case (Typed(_) | NullValued, Right(_)) => None + case (NotDateArithmetic, Right(_)) => None + case (NotDateArithmetic, Left(message)) if !dateArithmeticRefusal(message) => None + case (other, Left(message)) => Some(s"refused as [$message], expected $other") + }).map(m => s"[${c.expression}] $m") + } + info( + s"date arithmetic, DDL venue: ${ddlPopulation.size} computed columns; mismatches: ${mismatches.size}" + ) + withClue(mismatches.take(30).mkString("\n", "\n", "\n")) { + mismatches shouldBe empty + } + } + + private def resolvedItem(sql: String): (SingleSearch, Field) = + Parser(sql) match { + case Right(ss: SingleSearch) => + val resolved = ss.update(Some(schema)) + resolved -> resolved.select.fields + .find(_.fieldAlias.exists(_.alias == "x")) + .getOrElse(fail(s"[$sql] no x item")) + case other => fail(s"[$sql] $other") + } + + "a cast over date arithmetic" should "keep the type it casts to" in { + Seq( + "(d - d)::TIMESTAMP" -> SQLTypes.Timestamp, + "(d + 1)::TIMESTAMP" -> SQLTypes.Timestamp, + "(ts - ts)::BIGINT" -> SQLTypes.BigInt, + "(d + 1.5)::DATE" -> SQLTypes.Date, + "(MAX(d) - MIN(d))::DOUBLE" -> SQLTypes.Double + ).foreach { case (expression, t) => + val sql = + if (expression.contains("MAX")) s"SELECT g, $expression AS x FROM ev GROUP BY g" + else s"SELECT $expression AS x FROM ev" + val (resolved, item) = resolvedItem(sql) + withClue(s"[$expression] ") { + resolved.validateResolved() shouldBe Right(()) + item.identifier.reportedType shouldBe t + } + } + } + + "INTERVAL arithmetic and DATEDIFF" should "keep the types they had" in { + Seq( + "d - INTERVAL 1 DAY" -> SQLTypes.Timestamp, + "ts + INTERVAL 2 HOUR" -> SQLTypes.Timestamp, + "DATEDIFF(d, d)" -> SQLTypes.Numeric, + "TIMESTAMPDIFF(DAY, d, ts)" -> SQLTypes.Numeric + ).foreach { case (expression, t) => + Parser(s"SELECT $expression AS x FROM ev") match { + case Right(ss: SingleSearch) => + val resolved = ss.update(Some(schema)) + withClue(s"[$expression] ") { + resolved.validateResolved() shouldBe Right(()) + resolved.select.fields.head.identifier.reportedType shouldBe t + } + case other => fail(s"[$expression] $other") + } + } + } + + "a date field elasticsql never declared" should + "be a TIMESTAMP everywhere, from the declaration seam on" in { + // DESCRIBE reads the column's data type: the seam types it TIMESTAMP (the lead's ruling of + // 2026-10-05), while a declared DATE keeps its declaration + schema.find("u").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) + schema.find("d").map(_.dataType) shouldBe Some(SQLTypes.Date) + resolvedItem("SELECT u AS x FROM ev")._2.identifier.reportedType shouldBe SQLTypes.Timestamp + resolvedItem("SELECT d + 1 AS x FROM ev")._2.identifier.reportedType shouldBe SQLTypes.Date + // date arithmetic: a number of days keeps its time of day, a difference is fractional days + resolvedItem("SELECT u + 1 AS x FROM ev")._2.identifier.reportedType shouldBe + SQLTypes.Timestamp + resolvedItem("SELECT u - d AS x FROM ev")._2.identifier.reportedType shouldBe SQLTypes.Double + resolvedItem( + "SELECT g, MAX(u) - MIN(u) AS x FROM ev GROUP BY g" + )._2.identifier.reportedType shouldBe + SQLTypes.Double + // and a refusal names it as DESCRIBE does + resolvedItem("SELECT u * 2 AS x FROM ev")._1 + .validateResolved() + .swap + .getOrElse("") should include( + "cannot be applied to TIMESTAMP and BIGINT" + ) + } + + "CASE, GREATEST and LEAST" should "report the least common supertype of their branches" in { + resolvedItem( + "SELECT CASE WHEN n > 0 THEN d ELSE d2 END AS x FROM ev" + )._2.identifier.reportedType shouldBe SQLTypes.Date + resolvedItem( + "SELECT CASE WHEN n > 0 THEN d ELSE ts END AS x FROM ev" + )._2.identifier.reportedType shouldBe SQLTypes.Timestamp + resolvedItem("SELECT GREATEST(n, x) AS x FROM ev")._2.identifier.reportedType shouldBe + SQLTypes.Double + resolvedItem("SELECT LEAST(n, 2) AS x FROM ev")._2.identifier.reportedType shouldBe + SQLTypes.BigInt + // GREATEST / LEAST over dates and timestamps (the lead's ruling of 2026-10-05): a DATE for + // dates, a TIMESTAMP when they mix -- and a date difference of one and a date is days + Seq( + "SELECT GREATEST(d, d2) AS x FROM ev" -> SQLTypes.Date, + "SELECT LEAST(d, ts) AS x FROM ev" -> SQLTypes.Timestamp, + "SELECT GREATEST(d, CURRENT_DATE) AS x FROM ev" -> SQLTypes.Date, + "SELECT g, GREATEST(MAX(d), MAX(d2)) AS x FROM ev GROUP BY g" -> SQLTypes.Date, + "SELECT GREATEST(d, d2) - d AS x FROM ev" -> SQLTypes.BigInt, + "SELECT LEAST(d, ts) - d AS x FROM ev" -> SQLTypes.Double + ).foreach { case (sql, t) => + val (resolved, item) = resolvedItem(sql) + withClue(s"[$sql] ") { + resolved.validateResolved() shouldBe Right(()) + item.identifier.reportedType shouldBe t + } + } + // numbers and dates do not mix, as in every SQL engine + resolvedItem("SELECT GREATEST(d, n) AS x FROM ev")._1 + .validateResolved() + .swap + .getOrElse("") should include( + "GREATEST requires numeric arguments, or date and timestamp arguments, but got DATE for d " + + "and INT for n" + ) + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala index c29949b95..c15f814a0 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/HavingAggregateResolutionSpec.scala @@ -1,6 +1,8 @@ package app.softnetwork.elastic.sql.query import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -101,12 +103,32 @@ class HavingAggregateResolutionSpec extends AnyFlatSpec with Matchers { ).foreach { sql => withClue(s"[$sql] ") { Parser(sql).isRight shouldBe true + resolved(sql).validateResolved() shouldBe Right(()) } } - // A genuine mismatch stays loud, and names the ELEMENT types. - val msg = rejection("SELECT id FROM t GROUP BY id HAVING COUNT(x) IN ('a', 'b')") + // A genuine mismatch stays loud, and names the ELEMENT types -- once the column types are + // known (the lead's ruling of 2026-10-05: no TYPE refusal when the statement is parsed). + val sql = "SELECT id FROM t GROUP BY id HAVING COUNT(x) IN ('a', 'b')" + Parser(sql).isRight shouldBe true + val msg = + resolved(sql).validateResolved().swap.getOrElse(fail(s"expected a refusal for [$sql]")) msg should include("Type mismatch") msg should include("'BIGINT' is not compatible with 'VARCHAR'") - msg should not startWith Parser.InternalParseFailure } + + private val schema = SchemaTable( + "t", + columns = List( + Column("id", SQLTypes.Int), + Column("x", SQLTypes.Int), + Column("name", SQLTypes.Keyword), + Column("createdAt", SQLTypes.Timestamp) + ) + ) + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(single: SingleSearch) => single.update(Some(schema)) + case other => fail(s"[$sql] $other") + } } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala index 7fc3bea04..373bb6aa8 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/MixedTemporalComparisonSpec.scala @@ -373,17 +373,43 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr * operation to types [ZonedDateTime] and [ZonedDateTime]`). This project's standing rule: a * parse fix must not upgrade a loud failure into a silent corruption β€” or, as here, into a shard * error naming neither the column nor the SQL. + * + * ⚠️ The arithmetic caller is no longer one of them: a DATE minus a TIMESTAMP is date + * arithmetic, which reconciles its operands by its own rules (`ArithmeticExpression`, the lead's + * ruling of 2026-10-04) and answers the fractional days between them -- the row below. */ - "the date-carrying allowance" should "not leak to callers that cannot reconcile" in { - forAll( - Table( - "sql", - "SELECT name FROM t WHERE CAST(d AS DATE) BETWEEN CAST('2025-01-01' AS DATE) " + - "AND CAST('2025-12-31T00:00:00Z' AS TIMESTAMP)", - "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP)) AS x FROM t", - "SELECT CAST(d AS DATE) - CAST(ts AS TIMESTAMP) AS x FROM t" - ) - ) { sql => withClue(s"[$sql] ")(Parser(sql).isLeft shouldBe true) } + "the date-carrying allowance" should "reach BETWEEN and NULLIF, which now reconcile the pair" in { + // The ONE comparison rule (the lead's ruling of 2026-10-05): a DATE and a TIMESTAMP are + // comparable wherever a comparison is made, judged on the declared types. BETWEEN brings each + // bound to the operand's java.time type, as a single comparison does, and NULLIF compares the + // instants the two denote -- neither calls `isBefore` / `isEqual` across a `LocalDate` and a + // `ZonedDateTime`, which is what failed the shard. The NULLIF form is RUN in + // `DateArithmeticSpec`; the BETWEEN form is pinned here at RESOLVE only. + resolvedVerdict( + "SELECT name FROM t WHERE CAST(d AS DATE) BETWEEN CAST('2025-01-01' AS DATE) " + + "AND CAST('2025-12-31T00:00:00Z' AS TIMESTAMP)" + ) shouldBe Right(()) + resolvedVerdict("SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP)) AS x FROM t") shouldBe + Right(()) + // a TIME is no date-carrying temporal: still refused once the types are known, never at parse + val time = "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIME)) AS x FROM t" + Parser(time).isRight shouldBe true + resolvedVerdict(time).isLeft shouldBe true + } + + /** The post-resolution verdict of a statement: parsed, its schema attached, `validateResolved`. + */ + private def resolvedVerdict(sql: String): Either[String, Unit] = Parser(sql) match { + case Right(ss: SingleSearch) => ss.update(Some(schema)).validateResolved() + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + "a DATE minus a TIMESTAMP" should "be date arithmetic, the fractional days between them" in { + Parser("SELECT CAST(d AS DATE) - CAST(ts AS TIMESTAMP) AS x FROM t") match { + case Right(ss: SingleSearch) => + ss.select.fields.head.identifier.reportedType shouldBe SQLTypes.Double + case other => fail(s"$other") + } } "two operands cast to different date-carrying types" should "be comparable" in { @@ -403,11 +429,12 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr withClue(s"->\n$emitted\n")(emitted should include("atStartOfDay")) } - /** The other half of the same line: a TIME cast against a date-carrying cast is refused at PARSE, - * where both types are known without a schema. (A TIME against a bare COLUMN is only knowable - * after resolution β€” that is what `validateResolved` below is for.) + /** The other half of the same line: a TIME cast against a date-carrying cast is refused -- once + * the column types are known, like every TYPE rule (the lead's ruling of 2026-10-05); it used to + * be refused at PARSE, where both types are known without a schema. (A TIME against a bare + * COLUMN is only knowable after resolution β€” that is what `validateResolved` below is for.) */ - "a TIME cast against a date-carrying cast" should "be refused at parse" in { + "a TIME cast against a date-carrying cast" should "be refused once the types are known" in { forAll( Table( "sql", @@ -416,12 +443,15 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr "SELECT name FROM t WHERE CAST(ts AS TIMESTAMP) > CAST(ts AS TIME)" ) ) { sql => - withClue(s"[$sql] ")(Parser(sql).isLeft shouldBe true) + withClue(s"[$sql] ") { + Parser(sql).isRight shouldBe true + resolvedVerdict(sql).isLeft shouldBe true + } } // and TIME against TIME stays legal - Parser( + resolvedVerdict( "SELECT name FROM t WHERE CAST(ts AS TIME) > CAST('07:00:00' AS TIME)" - ).isRight shouldBe true + ) shouldBe Right(()) } /** πŸ”΄ `Expression.validate()` compares the types the two operands END UP with, not the ones their @@ -436,7 +466,9 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr * It is observable ONLY against a schema-resolved statement, which is what this asserts and why * the corpus could not falsify it. */ - "validate() on a schema-resolved statement" should "read the chain's types, not the column's" in { + "the comparison TYPE rule on a schema-resolved statement" should "read the chain's types, not the column's" in { + // `typeErrors`, not `validate()`: the TYPE rule moved off `validate()` (which checks syntax and + // structure only) to the post-resolution seam -- the lead's ruling of 2026-10-05. def criteriaOf(sql: String): Criteria = Parser(sql) match { case Right(ss: SingleSearch) => ss.update(Some(schema)).where.flatMap(_.criteria).getOrElse(fail(s"[$sql] no criteria")) @@ -446,11 +478,11 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr withClue("TIME = TIME ") { criteriaOf( "SELECT name FROM t WHERE CAST(ts AS TIME) = CAST('07:00:00' AS TIME)" - ).validate() shouldBe Right(()) + ).typeErrors shouldBe empty } // a TIME against a date-carrying column -- this was ACCEPTED (TIMESTAMP vs TIMESTAMP) withClue("TIME > TIMESTAMP ") { - criteriaOf("SELECT name FROM t WHERE CAST(ts AS TIME) > ts").validate().isLeft shouldBe true + criteriaOf("SELECT name FROM t WHERE CAST(ts AS TIME) > ts").typeErrors should not be empty } // and the date family stays comparable, resolved or not forAll( @@ -462,7 +494,12 @@ class MixedTemporalComparisonSpec extends AnyFlatSpec with Matchers with TableDr "SELECT name FROM t WHERE YEAR(d) = 2025", "SELECT name FROM t WHERE UPPER(name) = 'A'" ) - ) { sql => withClue(s"[$sql] ")(criteriaOf(sql).validate() shouldBe Right(())) } + ) { sql => + withClue(s"[$sql] ") { + criteriaOf(sql).validate() shouldBe Right(()) + criteriaOf(sql).typeErrors shouldBe empty + } + } } "a TIME operand" should "be refused against a date-carrying one, and accepted against a TIME" in { diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala index 32752b069..551724a50 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/NullIfOperandSpec.scala @@ -657,27 +657,34 @@ class NullIfOperandSpec extends AnyFlatSpec with Matchers with TableDrivenProper * `TemporalLiterals` -- the SAME machinery a WHERE clause asks, against the column's own mapping * `format` -- so the engine has ONE date-literal grammar, not two. */ - it should "ask the temporal-literal resolver about a string literal over a date column" in { + it should "read a well-formed date literal as that date, and ask the resolver about the rest" in { + // The ONE comparison rule (the lead's ruling of 2026-10-05): a string literal that spells a + // date or a timestamp, compared with a temporal, IS that temporal -- read here, as the instant + // it denotes, so no per-document `String -> temporal` coercion is needed in any venue. That is + // the half #382 recorded as its own story, and it is accepted now in every venue. forAll( Table( - // πŸ”΄ AMENDED by the #382 review: EVERY row is refused now, and the column records which - // JUDGE refused it, because that is what the wording has to come from. - // - // A temporal operand against a text one cannot be EMITTED -- the pair folds to VARCHAR, - // misses both typed arms and lands on `==`, which is silently `false` for every document. - // So a well-formed literal is refused too, with a message that says the engine cannot do - // it yet rather than pretending the SQL is wrong. A MALFORMED literal is still refused by - // the resolver, which names the literal and the field -- a strictly better message, and - // the reason the resolver stays reachable and goes first. - // - // Making the well-formed half WORK is its own story (a `String -> temporal` coercion per - // subtype, three venues, four ES majors; and `now-1d` is Elasticsearch date math, which - // has no Painless equivalent at all and must stay refused by name). + "expr", + "NULLIF(d, '2025-01-01')", + "NULLIF(d, '2025-01-01 10:00:00')", + "NULLIF(ts, '2025-01-01T10:00:00Z')", + "NULLIF('2025-01-01', d)" + ) + ) { expr => + everyVenueVerdict(expr, "DATE").foreach { case (venue, verdict) => + withClue(s"[$venue] [$expr] -> $verdict ") { + verdict shouldBe Right(()) + } + } + } + forAll( + Table( + // Every other literal beside a date is REFUSED, and the column records which JUDGE + // refused it, because that is what the wording has to come from. Elasticsearch date math + // (`now-1d`) and an epoch number spell no date: the engine refuses them by name. A + // MALFORMED literal is refused by the resolver, which names the literal and the field -- + // a strictly better message, and the reason the resolver stays reachable and goes first. ("expr", "judge"), - ("NULLIF(d, '2025-01-01')", "engine"), - ("NULLIF(d, '2025-01-01 10:00:00')", "engine"), - ("NULLIF(ts, '2025-01-01T10:00:00Z')", "engine"), - ("NULLIF('2025-01-01', d)", "engine"), ("NULLIF(d, 'now-1d')", "engine"), ("NULLIF(d, '1735689600000')", "engine"), ("NULLIF(d, 'x')", "resolver"), diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala index ccdb6fb32..396a16425 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessResidualsSpec.scala @@ -316,8 +316,11 @@ class PainlessResidualsSpec extends AnyFlatSpec with Matchers with TableDrivenPr // The single-chain processor emission must not move: one column, one chain, one parameter. processorOf("CREATE TABLE t (d DATE, c INTEGER SCRIPT AS (YEAR(d)))") shouldBe "def param1 = (ctx.d instanceof String ? " + + "(ctx.d.length() > 10 ? ZonedDateTime.parse((ctx.d).replace(\" \", \"T\"), " + + "DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + "LocalDate.parse((ctx.d).replace(\"/\", \"-\"), DateTimeFormatter.ofPattern(\"yyyy-MM-dd\"))" + - ".atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(ctx.d).atZone(ZoneId.of('Z')))" + + ".atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(ctx.d).atZone(ZoneId.of('Z')))" + ".get(ChronoField.YEAR); ctx.c = param1" } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessTargetSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessTargetSpec.scala new file mode 100644 index 000000000..61028a7b4 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/PainlessTargetSpec.scala @@ -0,0 +1,126 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType, PainlessTarget} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** A comparison between a `date` doc value and another date renders for the Elasticsearch major the + * client module gives the bridge (`PainlessTarget`). + * + * Elasticsearch 6.8 hands a query script a `date` doc value as a `JodaCompatibleZonedDateTime`, + * whose `isBefore` / `isAfter` / `isEqual` against a genuine `ZonedDateTime` fail with + * `Class.cast`; so on 6.8 each doc-reading operand of such a comparison is normalised to UTC + * first. From 7 on the doc value IS a `ZonedDateTime`: the scripts of those majors -- and of a + * rendering no client module gave a major to -- stay byte-for-byte what they were, which the byte + * pins of `ParameterIdentitySpec`, `PredicateTransformSurvivalSpec` and `PainlessResidualsSpec` + * hold for the default target. + */ +class PainlessTargetSpec extends AnyFlatSpec with Matchers { + + private val schema = SchemaTable( + "t", + columns = List( + Column("d", SQLTypes.Date), + Column("d2", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("n", SQLTypes.Int) + ) + ) + + private val normalisation = ".withZoneSameInstant(ZoneId.of('Z'))" + + private def occurrences(s: String): Int = + s.sliding(normalisation.length).count(_ == normalisation) + + private def predicateOf( + sql: String, + target: PainlessTarget, + contextType: PainlessContextType = PainlessContextType.Query + ): String = Parser(sql) match { + case Right(ss: SingleSearch) => + val ctx = PainlessContext(contextType, target) + val body = ss + .update(Some(schema)) + .where + .flatMap(_.criteria) + .map(_.painless(Some(ctx))) + .getOrElse(fail(s"[$sql] no WHERE criteria")) + s"$ctx$body" + case other => fail(s"[$sql] expected a SingleSearch, got $other") + } + + /** Item 1's comparison shapes: a doc value against a function, the clock, a literal or date + * arithmetic, in both directions -- each with how many of its operands read a document. + */ + private val shapes: Seq[(String, Int)] = Seq( + "SELECT n FROM t WHERE CAST(d AS TIMESTAMP) >= ts" -> 2, + "SELECT n FROM t WHERE LAST_DAY(d) = ts" -> 2, + "SELECT n FROM t WHERE CAST(ts AS DATE) = d" -> 2, + "SELECT n FROM t WHERE DATE_ADD(d, INTERVAL 1 DAY) > ts" -> 2, + "SELECT n FROM t WHERE DATE_ADD(d, INTERVAL 1 DAY) = d2" -> 2, + "SELECT n FROM t WHERE CURRENT_DATE > d" -> 1, + "SELECT n FROM t WHERE COALESCE(d, d2) > CAST('2024-01-01' AS DATE)" -> 1, + "SELECT n FROM t WHERE d2 = d + 1" -> 2, + "SELECT n FROM t WHERE d > d2 - 1" -> 2, + "SELECT n FROM t WHERE d >= CAST(ts AS DATE) - 1" -> 2 + ) + + "a comparison of a date doc value" should + "render as it always did for Elasticsearch 7, 8 and 9 and for no major at all" in { + shapes.foreach { case (sql, _) => + withClue(s"[$sql] ") { + val default = predicateOf(sql, PainlessTarget.Default) + Seq(7, 8, 9).foreach(major => predicateOf(sql, PainlessTarget(major)) shouldBe default) + } + } + } + + it should "normalise every operand that reads a document to UTC on Elasticsearch 6.8, and nothing else" in { + shapes.foreach { case (sql, documentOperands) => + withClue(s"[$sql] ") { + val es6 = predicateOf(sql, PainlessTarget(6)) + val default = predicateOf(sql, PainlessTarget.Default) + // one normalisation per operand that reads a document (date arithmetic, a cast, may + // already carry the same UTC conversion for reasons of their own: the DIFFERENCE is what + // the target adds) -- and nothing else differs from what the other majors render + occurrences(es6) - occurrences(default) shouldBe documentOperands + es6.replace(normalisation, "") shouldBe default.replace(normalisation, "") + } + } + } + + it should "not normalise outside a query: an ingest processor parses its own operands" in { + shapes.foreach { case (sql, _) => + withClue(s"[$sql] ") { + predicateOf(sql, PainlessTarget(6), PainlessContextType.Processor) shouldBe + predicateOf(sql, PainlessTarget.Default, PainlessContextType.Processor) + } + } + } + + "PainlessTarget" should "tell the major that hands Joda-compatible dates" in { + PainlessTarget(6).jodaCompatibleDates shouldBe true + Seq(7, 8, 9).foreach(major => PainlessTarget(major).jodaCompatibleDates shouldBe false) + PainlessTarget.Default.jodaCompatibleDates shouldBe false + PainlessContext().target shouldBe PainlessTarget.Default + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala index 86e4d7530..7c3658e82 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/ParameterIdentitySpec.scala @@ -291,7 +291,8 @@ class ParameterIdentitySpec extends AnyFlatSpec with Matchers { // `CAST(lastUpdated AS date) - INTERVAL 3 DAY` must key apart from the bare `lastUpdated` in // THEN, else THEN returns `lastUpdated - 3 days`. The interval lands on the CAST's parameter. // Executed on ES 8.18 (NOW = 2025-01-08): a -> createdAt 2025-01-01, d -> lastSeen + 2 = - // 2025-01-03, e -> lastUpdated = 2025-01-04 (the old pin answered 2025-01-01 for e). + // 2025-01-03, e -> lastUpdated = 2025-01-04 (the old pin answered 2025-01-01 for e) -- with + // the THEN values then converted to the operand's DATE; they are now the values themselves. // ⚠️ That is the NO-schema rendering, the one the bridge fixtures pin. The SCHEMA rendering of // this same statement fails the shard on `main` AND here (`Class.cast` at // `param2.isEqual(param5)`: `CURRENT_DATE - 7` is coerced to a `ZonedDateTime` against a @@ -301,20 +302,25 @@ class ParameterIdentitySpec extends AnyFlatSpec with Matchers { "SELECT CASE CURRENT_DATE - INTERVAL 7 DAY WHEN CAST(lastUpdated AS date) - INTERVAL 3 DAY THEN lastUpdated " + "WHEN lastSeen THEN lastSeen + INTERVAL 2 DAY ELSE createdAt END AS c, identifier FROM Table" ) + // The THEN value is the bare column, as it is: a simple CASE's value is typed by its THEN / + // ELSE values, never converted to its operand's DATE. emitted should include( "def param2 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate().minus(3, ChronoUnit.DAYS)); " + - "def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value.toLocalDate()); " + "def param3 = (doc['lastUpdated'].size() == 0 ? null : doc['lastUpdated'].value); " ) // The candidate is guarded too (issue #373): `lastSeen` is a column and a document may not // have it, which was an NPE before. emitted should endWith( "? param3 : param1 != null && param4 != null && param1.isEqual(param4) ? param5 : param6" ) - // Rule 2, normalisation FIRST: the branch's DATE narrowing was appended to this object before - // the chain rendered; the UTC normalisation must still come first (`LocalDate` has no - // `toInstant()`). + // Rule 2, normalisation FIRST: the UTC normalisation comes before the interval on the + // branch's own object (`LocalDate` has no `toInstant()`), and the WHEN candidate's DATE + // narrowing stays on a parameter of its own. emitted should include( - "doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).toLocalDate().plus(2, ChronoUnit.DAYS)" + "doc['lastSeen'].value.toInstant().atZone(ZoneId.of('Z')).plus(2, ChronoUnit.DAYS)" + ) + emitted should include( + "def param4 = (doc['lastSeen'].size() == 0 ? null : doc['lastSeen'].value.toLocalDate()); " ) } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala index e1a4ad85d..e194f2119 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TemporalLiteralsSpec.scala @@ -467,19 +467,20 @@ class TemporalLiteralsSpec extends AnyFlatSpec with Matchers { |}},"settings":{"index":{"number_of_shards":"1","number_of_replicas":"0"}}}}""".stripMargin val table = Index("events", json).schema - // πŸ”΄ DATE β€” the DECLARED type, preserved. An Elasticsearch `date` field IS a millisecond - // timestamp at runtime, but that is a different fact and it belongs to - // `SQLTypeUtils.runtimeType`, read via `GenericIdentifier.baseType`. Story 21.5 briefly mapped - // it here instead and the spurious `_meta` diff that caused cascaded into an unparseable ES 6.8 - // ALTER and a materialized-view slowdown. `Column.dataType` stays what the user wrote. - table.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Date) + // πŸ”΄ TIMESTAMP β€” this mapping carries no elasticsql declaration (no `_meta` data type), and a + // `date` field nobody declared holds instants: the lead's ruling of 2026-10-05, applied at the + // declaration seam (`schema.IndexField`). A DECLARED type is preserved verbatim, a DATE + // included (`ColumnMetaDiffSpec`): story 21.5 mapped every `date` to a timestamp in + // `SQLTypes.apply` instead, which rewrote declarations, and the spurious `_meta` diff that + // caused cascaded into an unparseable ES 6.8 ALTER and a materialized-view slowdown. + table.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) table.find("fmt_ts").flatMap(_.options.get("format")) shouldBe Some( StringValue("yyyy-MM-dd HH:mm:ss") ) // `date_nanos` is not a SQL type the mapping resolves to: it comes out as ANY, so it is never // a candidate. Pinned on purpose (AC 4: excluded, not covered). table.find("ts_nanos").map(_.dataType) shouldBe Some(SQLTypes.Any) - table.find("items.ts").map(_.dataType) shouldBe Some(SQLTypes.Date) + table.find("items.ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) whereSql( resolved("SELECT id FROM events WHERE event_ts >= '2026-06-04 00:00:00'", table) @@ -504,7 +505,7 @@ class TemporalLiteralsSpec extends AnyFlatSpec with Matchers { |"settings":{"index":{"number_of_shards":"1","number_of_replicas":"0"}}}""".stripMargin val single = Index("events_alias", s"""{"events":$events}""") single.name shouldBe "events_alias" - single.schema.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Date) + single.schema.find("event_ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) whereSql( resolved("SELECT id FROM events_alias WHERE event_ts >= '2026-06-04 00:00:00'", single.schema) ) shouldBe "WHERE event_ts >= '2026-06-04T00:00:00'" @@ -519,7 +520,7 @@ class TemporalLiteralsSpec extends AnyFlatSpec with Matchers { // a concrete index keyed by its own name is untouched by the rule Index("events", s"""{"events":$events}""").schema.find("event_ts").map(_.dataType) shouldBe - Some(SQLTypes.Date) + Some(SQLTypes.Timestamp) // R4-18: a member WITHOUT mappings still counts -- otherwise a two-index alias would resolve // silently to the other member (the silent-wrong-answer mode this story removes) diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala new file mode 100644 index 000000000..b20bf339a --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/query/TypeCheckResolutionSpec.scala @@ -0,0 +1,707 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.query + +import app.softnetwork.elastic.sql.{PainlessContext, PainlessContextType} +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.schema.{Column, Table => SchemaTable} +import app.softnetwork.elastic.sql.`type`.SQLTypes +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatest.prop.TableDrivenPropertyChecks + +/** No TYPE refusal before the column types are known (the lead's ruling of 2026-10-05). + * + * Every type rule `validate()` used to apply when the statement was PARSED -- where every column + * is `Any` -- is asked once the schema is attached (`SingleSearch.validateResolved`), on the real + * types. So: + * - parsing refuses no type, whatever the statement (with no schema, nothing is refused on types + * at all, and Elasticsearch answers); + * - each moved rule still refuses its TRUE mismatches, by name, once the types are known -- one + * row per rule below; + * - and the statements the parse-time rules refused although they are well typed now run: they + * are typed from the real columns. + */ +class TypeCheckResolutionSpec extends AnyFlatSpec with Matchers with TableDrivenPropertyChecks { + + // `SchemaTable`, because `TableDrivenPropertyChecks` shadows the name `Table`. + private val schema = SchemaTable( + "t", + columns = List( + Column("id", SQLTypes.Keyword), + Column("g", SQLTypes.Keyword), + Column("s", SQLTypes.Keyword), + Column("n", SQLTypes.Int), + Column("big", SQLTypes.BigInt), + Column("x", SQLTypes.Double), + Column("r", SQLTypes.Real), + Column("d", SQLTypes.Date), + Column("d2", SQLTypes.Date), + Column("ts", SQLTypes.Timestamp), + Column("b", SQLTypes.Boolean) + ) + ) + + private def parsed(sql: String): SingleSearch = Parser(sql) match { + case Right(single: SingleSearch) => single + case other => fail(s"[$sql] parsed as $other: no TYPE is refused at parse") + } + + private def resolved(sql: String): Either[String, Unit] = + parsed(sql).update(Some(schema)).validateResolved() + + "every TYPE rule moved off validate()" should + "refuse nothing at parse, and refuse its true mismatches by name once the types are known" in { + forAll( + Table( + ("rule", "sql", "refusal"), + // a comparison: its operands' match, TIME against a date + ( + "comparison", + "SELECT id FROM t WHERE n + 1 > 'abc'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ("comparison", "SELECT id FROM t WHERE s = n", "'KEYWORD' is not compatible with 'INT'"), + ("comparison (TIME)", "SELECT id FROM t WHERE CAST(ts AS TIME) > ts", "a TIME value"), + // IN: the operand against the list's element type + ( + "IN", + "SELECT id FROM t WHERE CAST(n AS INT) IN ('a', 'b')", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "IN (HAVING)", + "SELECT g FROM t GROUP BY g HAVING COUNT(n) IN ('a', 'b')", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + // BETWEEN: its bounds, and its operand against them + ( + "BETWEEN bounds", + "SELECT id FROM t WHERE n BETWEEN 1 AND 'a'", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + ( + "BETWEEN operand", + "SELECT id FROM t WHERE CAST(s AS INT) BETWEEN 'a' AND 'b'", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + // NULLIF: #382's rule, and the matching rule over the types the statement fixes + ("NULLIF (#382)", "SELECT NULLIF(s, 1) AS v FROM t", "compares KEYWORD with BIGINT"), + // (a DATE against a TIMESTAMP is comparable, the ONE comparison rule: TIME is not) + ( + "NULLIF (match)", + "SELECT NULLIF(CAST(d AS DATE), CAST(ts AS TIME)) AS v FROM t", + "output 'DATE' is not compatible with input 'TIME'" + ), + // CASE: boolean conditions, and a simple CASE's WHEN values against its operand + ( + "CASE WHEN", + "SELECT CASE WHEN n THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of type BOOLEAN" + ), + ( + "CASE WHEN", + "SELECT CASE n WHEN 'a' THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of the same type as the expression" + ), + // GREATEST / LEAST: numbers, or dates and timestamps -- one kind + ( + "GREATEST", + "SELECT GREATEST('a', 'b') AS v FROM t", + "GREATEST requires numeric arguments, or date and timestamp arguments, but got " + + "VARCHAR for 'a'" + ), + ( + "LEAST", + "SELECT LEAST(d, n) AS v FROM t", + "LEAST requires numeric arguments, or date and timestamp arguments, but got DATE for d " + + "and INT for n" + ), + // an arithmetic: date arithmetic's rules, and any other's operand match + ( + "date arithmetic", + "SELECT CURRENT_DATE * 2 AS v FROM t", + "operator * cannot be applied to DATE and BIGINT" + ), + ( + "date arithmetic (column)", + "SELECT d + d2 AS v FROM t", + "operator + cannot be applied to DATE and DATE" + ), + ( + "arithmetic", + "SELECT n + s AS v FROM t", + "output 'INT' is not compatible with input 'KEYWORD'" + ), + ("arithmetic (literals)", "SELECT 1 + 'abc' AS v FROM t", "input 'VARCHAR'") + ) + ) { (rule, sql, refusal) => + withClue(s"[$rule] [$sql] ") { + // nothing is refused at parse, and with no schema nothing is refused on types: the + // statement parses, and nothing else asks + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + } + + "the statements the parse-time rules refused although they are well typed" should + "be accepted once the column types are known" in { + forAll( + Table( + "sql", + "SELECT id FROM t WHERE CURRENT_DATE - d > 30", + "SELECT id FROM t WHERE d + 7 > CURRENT_DATE", + "SELECT id FROM t WHERE NOW() - ts > 1", + "SELECT id FROM t WHERE CAST(ts AS DATE) - d = 0", + "SELECT id FROM t WHERE d + 1 = CAST('2024-02-01' AS DATE)", + "SELECT id FROM t WHERE d + 1 > '2024-01-01'", + "SELECT g, MAX(d) + 1 AS v FROM t GROUP BY g HAVING v > CAST('2024-01-01' AS DATE)", + // a BOOLEAN column is a boolean condition -- refused at parse, where it was `Any` + "SELECT CASE WHEN b THEN 1 ELSE 0 END AS v FROM t", + // GREATEST / LEAST over dates and timestamps (the lead's ruling of 2026-10-05) + "SELECT GREATEST(d, d2) AS v FROM t", + "SELECT LEAST(d, ts, CURRENT_DATE) AS v FROM t", + "SELECT id FROM t WHERE GREATEST(d, d2) > CAST('2024-02-01' AS DATE)", + "SELECT g, GREATEST(MAX(d), MAX(d2)) AS v FROM t GROUP BY g" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + "a literal compared with a bare column" should + "be left to Elasticsearch, which coerces it against the column's mapping" in { + forAll( + Table( + "sql", + "SELECT id FROM t WHERE n = '5'", + "SELECT id FROM t WHERE d >= '2024-01-01'", + "SELECT id FROM t WHERE s = 5", + "SELECT id FROM t WHERE n IN ('1', '2')", + "SELECT id FROM t WHERE n BETWEEN '1' AND '9'" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + "a string literal compared with a temporal expression" should + "be the date or timestamp it spells, and refused when it spells none" in { + resolved("SELECT id FROM t WHERE d + 1 > '2024-01-01T10:00:00Z'") shouldBe Right(()) + resolved("SELECT id FROM t WHERE DATE_TRUNC(d, MONTH) >= '2024-01-01'") shouldBe Right(()) + resolved("SELECT id FROM t WHERE d + 1 > 'tomorrow'").swap.getOrElse("") should include( + "is not compatible with 'VARCHAR'" + ) + } + + /** A quoted number is read as a VALUE of the other operand's type, as PostgreSQL's input for that + * type reads it (the lead's ruling of 2026-10-05). Beside a whole-number type, `'1.5'` and + * `'1.0'` are no integer -- PostgreSQL 16: `invalid input syntax for type integer`, for a + * comparison and for `n + '1.5'` alike -- so the literal stays text and is refused by name at + * every site that reads one; beside DOUBLE, REAL or DECIMAL (a DOUBLE here) any number is one. + */ + "a quoted number beside a whole-number operand" should + "be a whole number, refused by name otherwise, and any number beside a fractional one" in { + forAll( + Table( + ("site", "sql", "refusal"), + ( + "comparison (INT)", + "SELECT id FROM t WHERE NULLIF(n, 0) = '1.5'", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "comparison (INT)", + "SELECT id FROM t WHERE NULLIF(n, 0) = '1.0'", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "comparison (BIGINT)", + "SELECT id FROM t WHERE COALESCE(big, 0) = '1.5'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "comparison (BIGINT)", + "SELECT id FROM t WHERE COALESCE(big, 0) = '1.0'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "CASE condition", + "SELECT id, CASE WHEN COALESCE(n, 0) = '1.5' THEN 1 ELSE 0 END AS v FROM t", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "an exponent", + "SELECT id FROM t WHERE NULLIF(n, 0) = '1e3'", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "past BIGINT", + "SELECT id FROM t WHERE COALESCE(big, 0) = '99999999999999999999'", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "IN", + "SELECT id FROM t WHERE NULLIF(n, 0) IN ('1', '1.5')", + "'INT' is not compatible with 'VARCHAR'" + ), + ( + "IN (HAVING)", + "SELECT g FROM t GROUP BY g HAVING MAX(big) IN ('1', '1.0')", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + ( + "BETWEEN", + "SELECT id FROM t WHERE COALESCE(big, 0) BETWEEN '1' AND '1.0'", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + ("NULLIF (INT)", "SELECT NULLIF(n, '1.5') AS v FROM t", "compares INT with VARCHAR"), + ( + "NULLIF (BIGINT)", + "SELECT NULLIF(big, '1.0') AS v FROM t", + "compares BIGINT with VARCHAR" + ), + ( + "CASE WHEN (INT)", + "SELECT CASE n WHEN '1.5' THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of the same type as the expression" + ), + ( + "CASE WHEN (BIGINT)", + "SELECT CASE big WHEN '1.0' THEN 1 ELSE 0 END AS v FROM t", + "CASE WHEN conditions must be of the same type as the expression" + ), + // a group filter compares the metric, which is no text + ( + "HAVING", + "SELECT g FROM t GROUP BY g HAVING MAX(n) > '4.5'", + "HAVING cannot be applied to MAX(n) > '4.5'" + ), + ( + "arithmetic (INT)", + "SELECT n + '1.5' AS v FROM t", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + ( + "arithmetic (INT, literal first)", + "SELECT '1.0' * n AS v FROM t", + "output 'VARCHAR' is not compatible with input 'INT'" + ), + ( + "arithmetic (BIGINT)", + "SELECT big - '1.0' AS v FROM t", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + ( + "arithmetic (per group)", + "SELECT g, SUM(n) + '1.5' AS v FROM t GROUP BY g", + "output 'INT' is not compatible with input 'VARCHAR'" + ) + ) + ) { (site, sql, refusal) => + withClue(s"[$site] [$sql] ") { + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + forAll( + Table( + "sql", + // a whole number is an integer, however it is written + "SELECT id FROM t WHERE NULLIF(n, 0) = '1'", + "SELECT id FROM t WHERE NULLIF(n, 0) = ' 1 '", + "SELECT id FROM t WHERE NULLIF(n, 0) = '+1'", + "SELECT id FROM t WHERE COALESCE(big, 0) = '10000000000'", + "SELECT CASE big WHEN '2' THEN 1 ELSE 0 END AS v FROM t", + "SELECT n + '1' AS v FROM t", + // beside DOUBLE, REAL or DECIMAL, any number + "SELECT id FROM t WHERE COALESCE(x, 0) = '1.5'", + "SELECT id FROM t WHERE COALESCE(x, 0) = '1e0'", + "SELECT id FROM t WHERE COALESCE(r, 0) = '1.0'", + "SELECT id FROM t WHERE CAST(s AS DECIMAL(10,2)) = '1.5'", + "SELECT id FROM t WHERE COALESCE(x, 0) IN ('1', '1.5')", + "SELECT id FROM t WHERE COALESCE(x, 0) BETWEEN '1.0' AND '1.5'", + "SELECT NULLIF(x, '1.5') AS v FROM t", + "SELECT CASE x WHEN '1.5' THEN 1 ELSE 0 END AS v FROM t", + "SELECT x + '1.5' AS v FROM t", + "SELECT g, AVG(n) + '1.5' AS v FROM t GROUP BY g", + "SELECT g FROM t GROUP BY g HAVING AVG(n) > '4.5'", + // beside a temporal a number is a number of days, whatever its fraction + "SELECT d + '1.5' AS v FROM t" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + /** The literal reading a comparison renders is the one its type rule judged: a DOUBLE operand + * compares the number a quoted `'1.5'` spells, and an integer one the whole number `'1'` spells. + */ + "a quoted number read beside a number" should "render as that number" in { + def criteriaPainless(sql: String): String = { + val search = parsed(sql).update(Some(schema)) + search.validateResolved() shouldBe Right(()) + search.where + .flatMap(_.criteria) + .map(_.painless(Some(PainlessContext(context = PainlessContextType.Query)))) + .getOrElse(fail(s"[$sql] no WHERE")) + } + criteriaPainless("SELECT id FROM t WHERE COALESCE(x, 0) = '1.5'") should include("== 1.5") + criteriaPainless("SELECT id FROM t WHERE COALESCE(big, 0) = '1'") should include("== 1L") + } + + /** The Painless a statement's one script field renders in a query, its declarations included. */ + private def fieldPainless(sql: String): String = { + val search = parsed(sql).update(Some(schema)) + search.validateResolved() shouldBe Right(()) + val field = search.scriptFields + .filterNot(_.isAggregation) + .headOption + .getOrElse(fail(s"[$sql] no script field")) + val ctx = PainlessContext(context = PainlessContextType.Query) + val body = field.painless(Some(ctx)) + s"$ctx$body" + } + + /** The ingest script of every computed column of a CREATE TABLE. */ + private def ingestPainless(ddl: String): String = Parser(ddl) match { + case Right(create: CreateTable) => + create.schema.columns.flatMap(_.script).map(_.source).mkString(" ;; ") + case other => fail(s"[$ddl] parsed as $other") + } + + /** NULLIF reads a string literal beside a temporal as the date or the timestamp it spells, and + * renders it from that READING: the literal's text is never parsed on a document -- by the + * operand's own format it could not be, for a date-time beside a DATE or a date beside a + * TIMESTAMP -- and a literal that is the answer is the value its reading denotes. + */ + "a NULLIF literal read as a date or a timestamp" should + "be rendered from its reading, never parsed on a document, in a query and a computed column" in { + forAll( + Table( + ("expression", "type", "value"), + ("NULLIF(d, '2024-01-31 10:00:00')", "DATE", None), + ("NULLIF(d, '2024-01-31T10:00:00Z')", "DATE", None), + ("NULLIF(ts, '2024-01-31')", "TIMESTAMP", None), + ("NULLIF(COALESCE(d, d2), '2024-01-31T10:00:00Z')", "DATE", None), + ("NULLIF('2024-01-31T10:00:00Z', d)", "DATE", Some("LocalDate.ofEpochDay(19753L)")), + ( + "NULLIF('2024-01-31', ts)", + "TIMESTAMP", + Some("Instant.ofEpochMilli(1706659200000L).atZone(ZoneId.of('Z'))") + ) + ) + ) { (expression, sqlType, value) => + Seq( + "query" -> fieldPainless(s"SELECT $expression AS v FROM t"), + "computed column" -> ingestPainless( + s"CREATE TABLE z (id INT, d DATE, d2 DATE, ts TIMESTAMP, " + + s"v $sqlType SCRIPT AS ($expression), PRIMARY KEY (id))" + ) + ).foreach { case (venue, script) => + withClue(s"[$venue] [$expression] $script ") { + script should not include "\"2024-01-31" + value.foreach(v => script should include(v)) + } + } + } + } + + /** `CASE b WHEN '1'` / `WHEN TRUE` compares with a boolean CONSTANT: rendered as the number it + * is, never as `(true ? 1L : 0L)`, a conditional over a constant, which Elasticsearch 6.8's + * Painless refuses to compile ("Extraneous conditional statement"). + */ + "a simple CASE over a BOOLEAN, compared with a boolean constant" should + "render the constant as a number, never as a conditional over a constant" in { + forAll( + Table( + ("value", "constant"), + ("'1'", "1L"), + ("'true'", "1L"), + ("TRUE", "1L"), + ("'0'", "0L"), + ("'false'", "0L"), + ("FALSE", "0L") + ) + ) { (value, constant) => + val script = fieldPainless(s"SELECT CASE b WHEN $value THEN 1 ELSE 0 END AS v FROM t") + withClue(s"[$value] $script ") { + script should not include "(true ?" + script should not include "(false ?" + script should include(s"= $constant;") + } + } + } + + /** A BOOLEAN column as a CASE condition is NULL on a document that has no value, and a NULL + * condition is not true: rendered `== true`, never as the bare value, which threw on that + * document. + */ + "a BOOLEAN column as a CASE condition" should + "render NULL as not true, in a query and in a computed column" in { + fieldPainless("SELECT CASE WHEN b THEN 1 ELSE 0 END AS v FROM t") should include( + "== true ? 1 : 0" + ) + ingestPainless( + "CREATE TABLE z (id INT, flag BOOLEAN, c INT SCRIPT AS (CASE WHEN flag THEN 1 ELSE 0 END), " + + "PRIMARY KEY (id))" + ) should include("== true ? 1 : 0") + } + + /** The type every SELECT item REPORTS -- what a `CREATE TABLE … AS SELECT` declares -- by alias. + */ + private def reported(sql: String): Map[String, String] = { + val search = parsed(sql).update(Some(schema)) + search.validateResolved() shouldBe Right(()) + search.select.fields.map { f => + f.fieldAlias.map(_.alias).getOrElse(f.identifier.sql) -> f.identifier.reportedType.typeId + }.toMap + } + + /** A NUMBER converted to a DATE is refused by name, as PostgreSQL refuses it (`cannot cast type + * integer to date`) -- in a query, per group and in a computed column; a number converted to a + * TIMESTAMP keeps core's epoch-millisecond conversion. + */ + "a number converted to a DATE" should "be refused by name, in a query and a computed column" in { + forAll( + Table( + ("sql", "refusal"), + ("SELECT (d - d2)::DATE AS v FROM t", "cannot cast BIGINT to DATE"), + ("SELECT CAST(n AS DATE) AS v FROM t", "cannot cast INT to DATE"), + ("SELECT n::DATE AS v FROM t", "cannot cast INT to DATE"), + ("SELECT CAST(x AS DATE) AS v FROM t", "cannot cast DOUBLE to DATE"), + ("SELECT CAST(1 AS DATE) AS v FROM t", "cannot cast BIGINT to DATE"), + ( + "SELECT CAST(YEAR(d) AS DATE) AS v FROM t", + "to DATE in expression: CAST(YEAR(d) AS DATE)" + ), + ("SELECT TRY_CAST(n AS DATE) AS v FROM t", "cannot cast INT to DATE"), + ("SELECT CONVERT(n, DATE) AS v FROM t", "cannot cast INT to DATE"), + ("SELECT id FROM t WHERE (d - d2)::DATE = d", "cannot cast BIGINT to DATE"), + ("SELECT g, (MAX(d) - MIN(d))::DATE AS v FROM t GROUP BY g", "cannot cast BIGINT to DATE") + ) + ) { (sql, refusal) => + withClue(s"[$sql] ") { + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + Parser( + "CREATE TABLE z (id INT, n INT, c DATE SCRIPT AS (CAST(n AS DATE)), PRIMARY KEY (id))" + ) match { + case Left(error) => + error.msg should include("Column 'c': Type mismatch: cannot cast INT to DATE") + case other => fail(s"a computed column CAST(n AS DATE) was accepted: $other") + } + forAll( + Table( + "sql", + "SELECT (d - d2)::TIMESTAMP AS v FROM t", + "SELECT CAST(n AS TIMESTAMP) AS v FROM t", + "SELECT (d - d2)::BIGINT AS v FROM t", + "SELECT CAST(ts AS DATE) AS v FROM t", + "SELECT CAST('2024-01-31' AS DATE) AS v FROM t" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + } + + /** A `CASE` or a `NULLIF` used as an operand is ONE value (issue #380): the operator applies to + * every branch -- never `c ? n : 0 + 1`, the ELSE branch plus one -- and a NULL it produces + * makes the result NULL rather than failing the script. + */ + "a CASE or a NULLIF used as an operand" should + "be rendered as one value, in a query and in a computed column" in { + val notNull = fieldPainless("SELECT (CASE WHEN n > 0 THEN 1 ELSE 0 END) + 1 AS v FROM t") + withClue(notNull) { + notNull should not include ": 0 + 1" + notNull should include("? 1 : 0) + 1") + } + Seq( + "SELECT CASE WHEN n > 0 THEN n ELSE 0 END + 1 AS v FROM t", + "SELECT (CASE WHEN n > 0 THEN n ELSE 0 END) + 1 AS v FROM t", + "SELECT CASE WHEN n > 0 THEN n END + 1 AS v FROM t" + ).foreach { sql => + val script = fieldPainless(sql) + withClue(s"[$sql] $script ") { + script should not include ": 0 + 1" + script should fullyMatch regex + """.*def (lv\d+) = \(.*\); \(\1 == null\) \? null : \(\1 \+ 1\)""" + } + } + val nullIf = fieldPainless("SELECT NULLIF(n, 3) + 1 AS v FROM t") + withClue(nullIf) { + nullIf should fullyMatch regex """.*\((param\d+) == null\) \? null : \(\1 \+ 1\)""" + } + val stored = ingestPainless( + "CREATE TABLE z (id INT, n INT, c BIGINT SCRIPT AS (CASE WHEN n > 0 THEN n ELSE 0 END + 1), " + + "c2 INT SCRIPT AS (NULLIF(n, 3) + 1), PRIMARY KEY (id))" + ) + withClue(stored) { + stored should not include ": 0 + 1" + stored should include("== null) ? null : (") + } + } + + /** `GREATEST` / `LEAST` over whole numbers keep the winner's whole type (issue #380): Painless's + * `Math.max` / `Math.min` take `double`s only. + */ + "GREATEST / LEAST over whole numbers" should "choose the winner by a comparison" in { + val greatest = fieldPainless("SELECT GREATEST(n, 2) AS v FROM t") + withClue(greatest) { + greatest should not include "Math.max" + greatest should include(">= 2 ?") + } + val least = fieldPainless("SELECT LEAST(n, big) AS v FROM t") + withClue(least) { + least should not include "Math.min" + least should include(" <= ") + } + fieldPainless("SELECT GREATEST(x, 2) AS v FROM t") should include("Math.max") + // two literals: the winner is the literal itself, converted to the BIGINT both are -- never a + // conditional over constants, which Elasticsearch 6.8 refuses as a whole script + val literals = fieldPainless("SELECT GREATEST(2, 4) AS v FROM t") + literals should endWith("((long) 4)") + literals should not include "?" + } + + /** `GREATEST` / `LEAST` answer the common type of ALL their arguments on every row, as PostgreSQL + * types them, and report it: each argument is read by the type it is declared with -- a `CAST` + * by the type it casts to, never by its column's -- and per document the value is converted to + * that type on every path, the argument that survives a NULL included. + */ + "GREATEST / LEAST" should "answer and report the common type of their arguments" in { + reported( + "SELECT GREATEST(CAST(n AS DOUBLE), 1) AS a, GREATEST(n, x) AS b, GREATEST(n, 2) AS c, " + + "GREATEST(n, CAST(n AS BIGINT)) AS d, LEAST(n, r) AS e, GREATEST(ABS(x), 1) AS f FROM t" + ) shouldBe Map( + "a" -> "DOUBLE", + "b" -> "DOUBLE", + "c" -> "BIGINT", + "d" -> "BIGINT", + "e" -> "REAL", + "f" -> "DOUBLE" + ) + // a CAST to DOUBLE is a DOUBLE: compared by Math.max, never as a whole number + val cast = fieldPainless("SELECT GREATEST(CAST(n AS DOUBLE), 1) AS v FROM t") + withClue(cast) { + cast should include("Math.max(") + cast should not include ">= 1 ?" + cast should include("((double) (") + } + // the argument that survives a NULL is converted with the winner: the value is read once + val survivor = fieldPainless("SELECT GREATEST(n, x) AS v FROM t") + withClue(survivor) { + survivor should fullyMatch regex + """.*def (lv\d+) = \(.*\); \(\1 == null \? \1 : \(def\) \(\(double\) \1\)\)""" + } + // a whole literal past the range of an int is a long literal beside a BIGINT + fieldPainless("SELECT GREATEST(n, 3000000000) AS v FROM t") should include( + ">= 3000000000L ? param1 : 3000000000L)" + ) + } + + /** `ABS` is typed by its argument, as PostgreSQL's `abs(integer) -> integer`: a quoted fraction + * beside `ABS` of an integer is refused by name, a whole number read; its value is that integer. + */ + "ABS" should "be typed by its argument" in { + reported("SELECT ABS(n) AS a, ABS(big) AS b, ABS(x) AS c FROM t") shouldBe Map( + "a" -> "INT", + "b" -> "BIGINT", + "c" -> "DOUBLE" + ) + forAll( + Table( + ("sql", "refusal"), + ("SELECT id FROM t WHERE ABS(n) = '1.5'", "'INT' is not compatible with 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(n) = '1.0'", "'INT' is not compatible with 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(big) = '1.5'", "'BIGINT' is not compatible with 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(n) IN ('1', '1.5')", "'INT' is not compatible with 'VARCHAR'"), + ( + "SELECT id FROM t WHERE ABS(n) BETWEEN '1' AND '1.5'", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + ("SELECT ABS(n) + '1.5' AS v FROM t", "is not compatible with input 'VARCHAR'"), + ("SELECT id FROM t WHERE ABS(n) + '1.5' > 0", "ABS(n) + '1.5'"), + ( + "SELECT NULLIF(ABS(n), '1.5') AS v FROM t", + "with VARCHAR: NULLIF requires two arguments of comparable types" + ), + ( + "SELECT CASE WHEN ABS(n) = '1.5' THEN 1 ELSE 0 END AS v FROM t", + "'INT' is not compatible with 'VARCHAR'" + ) + ) + ) { (sql, refusal) => + withClue(s"[$sql] ") { + parsed(sql) + resolved(sql).swap.getOrElse(fail("accepted once the types are known")) should include( + refusal + ) + } + } + forAll( + Table( + "sql", + "SELECT id FROM t WHERE ABS(n) = '2'", + "SELECT id FROM t WHERE ABS(x) = '1.5'", + "SELECT id FROM t WHERE ABS(n) > 2" + ) + ) { sql => withClue(s"[$sql] ")(resolved(sql) shouldBe Right(())) } + fieldPainless("SELECT ABS(n) AS v FROM t") should not include "Math.abs" + fieldPainless("SELECT ABS(x) AS v FROM t") should include("Double.valueOf(Math.abs(") + } + + /** `NULLIF(, x)` is typed by the literal's READING beside `x` -- what it answers + * and what PostgreSQL types it -- never by the VARCHAR the literal is written as. + */ + "NULLIF with a quoted literal first" should "be typed by the literal's reading" in { + reported( + "SELECT NULLIF('2024-01-31', ts) AS a, NULLIF('2024-01-31T10:00:00Z', d) AS b, " + + "NULLIF('3', n) AS c, NULLIF('1.5', x) AS e, NULLIF('true', b) AS f, " + + "NULLIF(ts, '2024-01-31') AS h FROM t" + ) shouldBe Map( + "a" -> "TIMESTAMP", + "b" -> "DATE", + "c" -> "BIGINT", + "e" -> "DOUBLE", + "f" -> "BOOLEAN", + "h" -> "TIMESTAMP" + ) + } + + /** A simple `CASE` answers its `THEN` / `ELSE` values in their own type: the operand is only + * compared, in the type it is compared in, and never converts the results. + */ + "a simple CASE" should "answer its THEN / ELSE values in their own type" in { + reported( + "SELECT CASE s WHEN 'a' THEN 1 ELSE 0 END AS a, CASE x WHEN 1.5 THEN 1 ELSE 0 END AS b FROM t" + ) shouldBe Map("a" -> "BIGINT", "b" -> "BIGINT") + val overText = fieldPainless("SELECT CASE s WHEN 'a' THEN 1 ELSE 0 END AS v FROM t") + withClue(overText) { + overText should include(".compareTo(") + overText should endWith("? 1 : 0") + } + val overDouble = fieldPainless("SELECT CASE x WHEN 1.5 THEN 1 ELSE 0 END AS v FROM t") + withClue(overDouble)(overDouble should endWith("? 1 : 0")) + val stored = ingestPainless( + "CREATE TABLE z (id INT, s KEYWORD, c BIGINT SCRIPT AS (CASE s WHEN '1' THEN 1 ELSE 0 END), " + + "PRIMARY KEY (id))" + ) + withClue(stored)(stored should include("? 1 : 0")) + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala index 3d3978125..3de5a7b61 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/DdlScriptSchemaSpec.scala @@ -99,14 +99,18 @@ class DdlScriptSchemaSpec extends AnyFlatSpec with Matchers { // answered that question one way, hard-coded and untested, and was wrong for every documented // example. Verified as an ingest pipeline on ES 6.8.23, 7.17.29, 8.18.3 and 9.0.3 against // {"created":"2025-01-10"}, {"created":"2025/01/10"} and {"created":1736467200000}: all three - // store `y: 2025`. + // store `y: 2025`. A string longer than a date is a date-time (Elasticsearch accepts one into a + // DATE column): its UTC day, where the strict date parse threw and stored nothing. createSource( "CREATE TABLE t (created DATE, y INTEGER SCRIPT AS (YEAR(created)))", "y" ) shouldBe "def param1 = (ctx.created instanceof String ? " + + """(ctx.created.length() > 10 ? ZonedDateTime.parse((ctx.created).replace(" ", "T"), """ + + "DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + """LocalDate.parse((ctx.created).replace("/", "-"), """ + - """DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z')) """ + + """DateTimeFormatter.ofPattern("yyyy-MM-dd")).atStartOfDay(ZoneId.of('Z'))) """ + ": Instant.ofEpochMilli(ctx.created).atZone(ZoneId.of('Z'))).get(ChronoField.YEAR); " + "ctx.y = param1" } diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala index 1dac75888..71399bd17 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/IngestTemporalSpec.scala @@ -157,15 +157,20 @@ class IngestTemporalSpec extends AnyFlatSpec with Matchers with TableDrivenPrope // CURRENT_DATE)`: the whole years elapsed from `birthdate` to the start of `CURRENT_DATE` // (UTC), which the processor holds as the current instant and reads as a date. Run as an // ingest pipeline on ES 8.18.3, `{"birthdate":"1990-05-20"}` and `{"birthdate":643161600000}` - // both store that age. + // both store that age. A date-time string in the DATE column (Elasticsearch accepts one) is + // read as the UTC day of the instant it spells -- the arm `DateArithmeticSpec`'s computed + // columns execute -- where the strict date parse threw and stored nothing. source( "CREATE TABLE t (birthdate DATE, age INTEGER SCRIPT AS (TIMESTAMPDIFF(YEAR, birthdate, CURRENT_DATE)))", "age" ) shouldBe "def param1 = ctx.birthdate; " + "def param2 = (param1 instanceof String ? " + + """(param1.length() > 10 ? ZonedDateTime.parse((param1).replace(" ", "T"), """ + + "DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of('Z'))).withZoneSameInstant(ZoneId.of('Z'))" + + ".truncatedTo(ChronoUnit.DAYS) : " + """LocalDate.parse((param1).replace("/", "-"), DateTimeFormatter.ofPattern("yyyy-MM-dd"))""" + - ".atStartOfDay(ZoneId.of('Z')) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); " + + ".atStartOfDay(ZoneId.of('Z'))) : Instant.ofEpochMilli(param1).atZone(ZoneId.of('Z'))); " + "def param3 = ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), " + "ZoneId.of('Z')); " + "def param4 = LocalDate.from(param3).atStartOfDay(ZoneId.of('Z')); " + diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/schema/MergedSchemaSpec.scala b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/MergedSchemaSpec.scala new file mode 100644 index 000000000..0e20ae572 --- /dev/null +++ b/sql/src/test/scala/app/softnetwork/elastic/sql/schema/MergedSchemaSpec.scala @@ -0,0 +1,179 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.sql.schema + +import app.softnetwork.elastic.schema.Index +import app.softnetwork.elastic.sql.StringValue +import app.softnetwork.elastic.sql.parser.Parser +import app.softnetwork.elastic.sql.query.SingleSearch +import app.softnetwork.elastic.sql.`type`.SQLTypes +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.collection.immutable.ListMap + +/** The schema an index PATTERN or a comma list answers with (`mergedSchema`): the union of the + * matched indices' columns, merged FIELD BY FIELD -- a field the indices map alike keeps its type, + * a field they map differently is left untyped (`Any`), so nothing is refused on it and + * Elasticsearch answers for it, while every other field is typed as over a single index. + */ +class MergedSchemaSpec extends AnyFlatSpec with Matchers { + + private def index(name: String, properties: String): (String, Table) = + name -> Index(name, s"""{"mappings":{"properties":{$properties}},"settings":{}}""").schema + + "indices that agree" should "answer the union of their columns" in { + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index("logs-1", """"ts":{"type":"date"},"g":{"type":"keyword"}"""), + index("logs-2", """"ts":{"type":"date"},"n":{"type":"long"}""") + ) + ) + disagreements shouldBe empty + merged.name shouldBe "logs-*" + merged.columns.map(c => c.name -> c.dataType) should contain theSameElementsAs Seq( + "g" -> SQLTypes.Keyword, + "n" -> SQLTypes.BigInt, + // a `date` nobody declared is a TIMESTAMP at the declaration seam, and stays one merged + "ts" -> SQLTypes.Timestamp + ) + } + + it should "agree on a TIMESTAMP one index declares and another holds undeclared" in { + val declared = Index( + "logs-1", + """{"mappings":{"_meta":{"columns":{"ts":{"data_type":"TIMESTAMP","not_null":"false"}}},""" + + """"properties":{"ts":{"type":"date"}}},"settings":{}}""" + ).schema + val (merged, disagreements) = + mergedSchema("logs-*", Seq("logs-1" -> declared, index("logs-2", """"ts":{"type":"date"}"""))) + disagreements shouldBe empty + merged.find("ts").map(_.dataType) shouldBe Some(SQLTypes.Timestamp) + } + + "indices that disagree on a field" should "leave that field alone untyped, and type every other" in { + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index("logs-1", """"ts":{"type":"date"},"c":{"type":"date"},"g":{"type":"keyword"}"""), + index("logs-2", """"ts":{"type":"date"},"c":{"type":"keyword"},"n":{"type":"long"}""") + ) + ) + disagreements shouldBe Seq("'c' (KEYWORD, TIMESTAMP)") + merged.columns.map(c => c.name -> c.dataType) should contain theSameElementsAs Seq( + "c" -> SQLTypes.Any, + "g" -> SQLTypes.Keyword, + "n" -> SQLTypes.BigInt, + "ts" -> SQLTypes.Timestamp + ) + } + + it should "count a date's format as part of its mapping: a temporal literal is read against it" in { + val withFormat = Table( + "logs-2", + columns = List( + Column( + "ts", + SQLTypes.Timestamp, + options = ListMap("format" -> StringValue("epoch_millis")) + ), + Column("g", SQLTypes.Keyword) + ) + ) + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index("logs-1", """"ts":{"type":"date"},"g":{"type":"keyword"}"""), + "logs-2" -> withFormat + ) + ) + disagreements shouldBe Seq("'ts' (TIMESTAMP)") + merged.find("ts").map(_.dataType) shouldBe Some(SQLTypes.Any) + merged.find("ts").flatMap(_.options.get("format")) shouldBe None + merged.find("g").map(_.dataType) shouldBe Some(SQLTypes.Keyword) + } + + it should "merge sub-fields by the same rule, under their parent" in { + val (merged, disagreements) = mergedSchema( + "logs-*", + Seq( + index( + "logs-1", + """"s":{"type":"text","fields":{"raw":{"type":"keyword"},"n":{"type":"long"}}}""" + ), + index( + "logs-2", + """"s":{"type":"text","fields":{"raw":{"type":"keyword"},"n":{"type":"keyword"},"w":{"type":"keyword"}}}""" + ) + ) + ) + disagreements shouldBe Seq("'s.n' (BIGINT, KEYWORD)") + merged.find("s").map(_.dataType) shouldBe Some(SQLTypes.Text) + merged.find("s.raw").map(_.dataType) shouldBe Some(SQLTypes.Keyword) + merged.find("s.w").map(_.dataType) shouldBe Some(SQLTypes.Keyword) + merged.find("s.n").map(_.dataType) shouldBe Some(SQLTypes.Any) + } + + /** The pattern of `DateArithmeticSpec`: two indices that agree on every field but `c`. */ + private lazy val pattern: Schema = + mergedSchema( + "ev_*", + Seq( + index( + "ev_a", + """"g":{"type":"keyword"},"d":{"type":"date"},"d2":{"type":"date"},"ts":{"type":"date"},"n":{"type":"integer"},"c":{"type":"date"}""" + ), + index("ev_b", """"g":{"type":"keyword"},"d":{"type":"date"},"c":{"type":"keyword"}""") + ) + )._1 + + private def resolved(sql: String): SingleSearch = + Parser(sql) match { + case Right(ss: SingleSearch) => ss.update(Some(pattern)) + case other => fail(s"[$sql] $other") + } + + "a statement over a pattern with one conflicting field" should + "type every field the indices agree on, and refuse nothing on the conflicting one" in { + // the fields they agree on: the type rules apply as over a single index (`d` is a `date` + // elasticsql never declared -- a TIMESTAMP -- so a difference of two is fractional days) + resolved("SELECT g, MAX(d) - MIN(d) AS x FROM ev_* GROUP BY g").select.fields + .find(_.fieldAlias.exists(_.alias == "x")) + .map(_.identifier.reportedType) shouldBe Some(SQLTypes.Double) + resolved("SELECT d * 2 AS x FROM ev_*").validateResolved().swap.getOrElse("") should include( + "operator * cannot be applied to TIMESTAMP and BIGINT" + ) + resolved("SELECT n + g AS x FROM ev_*").validateResolved().isLeft shouldBe true + // the conflicting field: no type, so no type rule refuses it -- Elasticsearch answers + Seq( + "SELECT c * 2 AS x FROM ev_*", + "SELECT c + d AS x FROM ev_*", + "SELECT n FROM ev_* WHERE c = 1", + "SELECT n FROM ev_* WHERE c + 1 > CURRENT_DATE", + "SELECT g, MAX(c) - MIN(c) AS x FROM ev_* GROUP BY g" + ).foreach { sql => + withClue(s"[$sql] ") { + resolved(sql).validateResolved() shouldBe Right(()) + } + } + resolved( + "SELECT c + 1 AS x FROM ev_*" + ).select.fields.head.identifier.reportedType.isUnknown shouldBe + true + } +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala new file mode 100644 index 000000000..f39e830cb --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/DateArithmeticSpec.scala @@ -0,0 +1,2712 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Sink, Source} +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ + ElasticFailure, + ElasticSuccess, + QueryRows, + QueryStream, + QueryStructured +} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.persistence.generateUUID +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.time.temporal.ChronoUnit +import java.time.{Instant, LocalDate, LocalDateTime, OffsetDateTime, ZoneOffset, ZonedDateTime} +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.language.implicitConversions +import scala.util.Try + +/** Date arithmetic EXECUTES and answers what SQL engines answer, in every venue a script runs in: a + * row SELECT, a WHERE, a per-group SELECT (`bucket_script`), a HAVING over its alias + * (`bucket_selector`) and a computed column (`SCRIPT AS`, an ingest processor) -- the STORED + * value. + * + * The rules (the lead's ruling of 2026-10-04): DATE - DATE is the calendar days between the two; a + * difference with a TIMESTAMP on either side is the milliseconds between, in days; a DATE moved by + * a whole number of days is a DATE, by a fractional one an instant whose fraction is a time of + * day; a TIMESTAMP moved by any number of days is a TIMESTAMP; a NULL operand gives NULL; and the + * other combinations are refused by name -- once the column types are known, before anything runs + * (the lead's ruling of 2026-10-05: no type is refused when a statement is parsed). + * + * Every expected value is computed HERE, with `java.time`, from the fixture -- never read from the + * engine. The fixture holds a group with two dates, groups with one, and a non-empty group where + * the date columns are absent; DATE and TIMESTAMP columns, the timestamps carrying a time of day; + * negative differences; January 31 to February 1 (1 day) and 2024-02-28 to 2024-03-01 (2 days, + * across a leap day); dates before 1970; dates written as epoch-millisecond NUMBERS; and rows + * where an operand is NULL. The computed-column table also stores a DATE column written as a + * date-TIME string, which Elasticsearch accepts. + * + * Also measured here, each in its own test: an index created by a bulk load with no declaration (a + * `date` keeps its time), an index PATTERN (it answers like its index), the request clock in a + * per-group calculation, casts over arithmetic (NULL-safe, applied), and every type rule that + * waits for the column types (refused by name, never at parse; with no schema, never by core). + */ +trait DateArithmeticSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + private val table = "date_arithmetic" + + private val computedTable = "date_arithmetic_computed" + + private val rawIndex = "date_arithmetic_raw" + + private val patternA = "date_arithmetic_pat_a" + + private val patternB = "date_arithmetic_pat_b" + + private val disagreeA = "date_arithmetic_dis_a" + + private val disagreeB = "date_arithmetic_dis_b" + + /** The documents of [[table]] with every `d` written as a date-TIME string of its UTC day. */ + private val dateTimeTable = "date_arithmetic_dt" + + /** The comparison rule's fixture (`"the ONE comparison rule"`). */ + private val comparisonTable = "date_arithmetic_cmp" + + /** A quoted number beside an INT, a BIGINT and a DOUBLE (`"a quoted number"`). */ + private val numberTable = "date_arithmetic_num" + + /** An index created by a bulk load, then altered through elasticsql. */ + private val alteredRawIndex = "date_arithmetic_raw_altered" + + /** A table elasticsql created, to which a bulk load later adds a `date` field. */ + private val grownTable = "date_arithmetic_grown" + + /** The indices of the pattern whose cached schema DDL must refresh. */ + private val refreshedA = "date_arithmetic_rf_a" + private val refreshedB = "date_arithmetic_rf_b" + + /** NULLIF with a date or a timestamp literal (`"NULLIF with a date or a timestamp literal"`). */ + private val nullifTable = "date_arithmetic_nullif" + + /** A simple CASE over a BOOLEAN (`"a simple CASE over a BOOLEAN"`). */ + private val booleanTable = "date_arithmetic_bool" + + /** A BOOLEAN column as a CASE condition (`"a CASE WHEN over a BOOLEAN column"`). */ + private val flagTable = "date_arithmetic_flag" + + /** PostgreSQL's fixture for the operand tests (`"a CASE or a NULLIF used as an operand"` and the + * tests after it), and the computed-column tables and `CREATE TABLE … AS SELECT` targets they + * create. + */ + private val operandName = "date_arithmetic_operand" + private val operandTables = Seq( + operandName, + "date_arithmetic_operand_cc", + "date_arithmetic_greatest_cc", + "date_arithmetic_greatest_type_cc", + "date_arithmetic_greatest_long_cc", + "date_arithmetic_greatest_double", + "date_arithmetic_greatest_cast", + "date_arithmetic_greatest_whole", + "date_arithmetic_tocast", + "date_arithmetic_tocast_ctas", + "date_arithmetic_abs_cc", + "date_arithmetic_abs_ctas", + "date_arithmetic_nullif_ts", + "date_arithmetic_nullif_d", + "date_arithmetic_nullif_n", + "date_arithmetic_case_cc", + "date_arithmetic_case_ctas" + ) + + private val DayMillis: Long = 86400000L + + /** One document: `None` is a column the document does not carry. + * + * @param epoch + * the dates and timestamps are written as epoch-millisecond NUMBERS + * @param dText + * `d` is written as this text -- a date-time string in a DATE column -- while `d` is the UTC + * day it falls in + */ + private final case class Row( + id: String, + g: String, + d: Option[LocalDate], + d2: Option[LocalDate], + ts: Option[Instant], + ts2: Option[Instant], + n: Option[Int], + x: Option[Double], + epoch: Boolean = false, + dText: Option[String] = None + ) + + private def date(s: String): Option[LocalDate] = Some(LocalDate.parse(s)) + private def instant(s: String): Option[Instant] = Some(Instant.parse(s)) + + private val rows: Seq[Row] = Seq( + Row( + "r1", + "g1", + date("2024-01-31"), + date("2024-02-01"), + instant("2024-01-31T10:30:00Z"), + instant("2024-02-01T22:15:00Z"), + Some(3), + Some(0.25) + ), + Row( + "r2", + "g1", + date("2024-03-01"), + date("2024-02-28"), + instant("2024-03-01T06:00:00Z"), + instant("2024-02-28T18:45:30.500Z"), + Some(-2), + Some(1.5) + ), + Row( + "r3", + "g2", + date("2023-12-25"), + None, + instant("2023-12-25T23:59:59.999Z"), + None, + None, + None + ), + Row("r4", "g3", None, None, None, None, Some(1), Some(2.0)), + Row( + "r5", + "g4", + date("2024-02-29"), + date("2024-03-01"), + instant("2024-02-29T12:00:00Z"), + instant("2024-03-01T00:00:00Z"), + Some(10), + Some(-0.5) + ), + // before 1970: negative epochs, whose days must FLOOR, not truncate toward zero + Row( + "r6", + "g5", + date("1969-12-31"), + date("1960-02-29"), + instant("1969-12-31T23:00:00Z"), + instant("1970-01-01T01:00:00Z"), + Some(-1), + Some(-0.25) + ), + // written as epoch-millisecond NUMBERS, a negative one included + Row( + "r7", + "g6", + date("2024-01-31"), + date("1969-12-31"), + instant("2024-01-31T10:30:00Z"), + instant("1969-12-31T23:00:00Z"), + Some(2), + Some(0.5), + epoch = true + ) + ) + + /** Stored by the computed-column table only: a DATE column holding a date-TIME string, which + * Elasticsearch accepts into a `date` field and which a DATE operand reads as its UTC day. + */ + private val computedOnlyRows: Seq[Row] = Seq( + Row( + "r8", + "g7", + date("2024-01-31"), + date("2024-02-01"), + instant("2024-01-31T22:30:00Z"), + instant("2024-02-01T01:30:00Z"), + Some(1), + Some(1.0), + dText = Some("2024-01-31T22:30:00Z") + ) + ) + + private val columns = + "id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, ts2 TIMESTAMP, n INT, x DOUBLE" + + /** The computed columns, each with the type the rules give its expression. */ + private val computed: Seq[(String, String, String)] = Seq( + ("c_dd", "BIGINT", "d - d2"), + ("c_dd2", "BIGINT", "d2 - d"), + ("c_tt", "DOUBLE", "ts - ts2"), + ("c_dt", "DOUBLE", "d - ts"), + ("c_dp1", "DATE", "d + 1"), + ("c_dm1", "DATE", "d - 1"), + ("c_nd", "DATE", "n + d"), + ("c_dp15", "TIMESTAMP", "d + 1.5"), + ("c_dx", "TIMESTAMP", "d + x"), + ("c_tp1", "TIMESTAMP", "ts + 1"), + ("c_tm15", "TIMESTAMP", "ts - 1.5"), + ("c_null", "DATE", "d + NULL"), + // ---- the fix round: operands that are not bare columns, a numeric string, casts ---- + ("c_coal", "BIGINT", "COALESCE(d, d2) - d2"), + ("c_nullif", "BIGINT", "NULLIF(d, d2) - d2"), + ("c_case", "BIGINT", "CASE WHEN n > 0 THEN d ELSE d2 END - d"), + ("c_castd", "BIGINT", "CAST(ts AS DATE) - d"), + ("c_str", "DATE", "d + '1'"), + ("c_tsbig", "BIGINT", "(ts - ts2)::BIGINT"), + ("c_nxbig", "BIGINT", "(n + x)::BIGINT"), + ("c_xint", "INT", "(x * 2)::INT"), + ("c_ddts", "TIMESTAMP", "(d - d2)::TIMESTAMP"), + ("c_tsts", "TIMESTAMP", "(ts - ts2)::TIMESTAMP"), + // ---- fix round 2: GREATEST / LEAST over dates, a number cast to a TIMESTAMP ---- + ("c_gr", "DATE", "GREATEST(d, d2)"), + ("c_le", "TIMESTAMP", "LEAST(d, ts)"), + ("c_grd", "BIGINT", "GREATEST(d, d2) - d"), + ("c_nts", "TIMESTAMP", "CAST(n AS TIMESTAMP)"), + ("c_xts", "TIMESTAMP", "CAST(x AS TIMESTAMP)") + ) + + private def json(r: Row): String = { + def dateValue(v: LocalDate): String = + if (r.epoch) s"${start(v).toEpochMilli}" else s""""$v"""" + def instantValue(v: Instant): String = if (r.epoch) s"${v.toEpochMilli}" else s""""$v"""" + (Seq(s""""id":"${r.id}"""", s""""g":"${r.g}"""") ++ + r.dText.map(t => s""""d":"$t"""").orElse(r.d.map(v => s""""d":${dateValue(v)}""")) ++ + r.d2.map(v => s""""d2":${dateValue(v)}""") ++ + r.ts.map(v => s""""ts":${instantValue(v)}""") ++ + r.ts2.map(v => s""""ts2":${instantValue(v)}""") ++ + r.n.map(v => s""""n":$v""") ++ r.x.map(v => s""""x":$v""")).mkString("{", ",", "}") + } + + private def bulk(index: String, docs: List[String]): Unit = { + implicit val bulkOptions: BulkOptions = BulkOptions(defaultIndex = index, logEvery = 100) + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + client.bulk[String](docs, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(_) => client.refresh(index) + case ElasticFailure(error) => + fail(s"Bulk indexing into $index failed: ${error.message}") + } + } + + override def beforeAll(): Unit = { + super.beforeAll() + run(s"CREATE TABLE $table ($columns, PRIMARY KEY (id))") + run( + s"CREATE TABLE $computedTable ($columns, " + + computed.map { case (c, t, e) => s"$c $t SCRIPT AS ($e)" }.mkString(", ") + + ", PRIMARY KEY (id))" + ) + bulk(table, rows.map(json).toList) + bulk(computedTable, (rows ++ computedOnlyRows).map(json).toList) + } + + override def afterAll(): Unit = { + (Seq( + table, + computedTable, + rawIndex, + patternA, + patternB, + disagreeA, + disagreeB, + dateTimeTable, + comparisonTable, + numberTable, + alteredRawIndex, + grownTable, + refreshedA, + refreshedB, + nullifTable, + booleanTable, + flagTable + ) ++ operandTables).foreach(t => + Try(Await.result(client.run(s"DROP TABLE IF EXISTS $t"), 60.seconds)) + ) + super.afterAll() + } + + private def run(sql: String): Unit = + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(_) => () + case ElasticFailure(error) => fail(s"[$sql] failed: ${error.message}") + } + + /** The rows of a statement, through `GatewayApi.run` (JDBC / REPL / Flight SQL). */ + private def gatewayRows(sql: String): Either[String, Seq[ListMap[String, Any]]] = + Try { + Await.result(client.run(sql), 60.seconds) match { + case ElasticSuccess(QueryRows(rs, _)) => Right(rs) + case ElasticSuccess(QueryStructured(response, _)) => Right(response.results) + case ElasticSuccess(QueryStream(stream, _)) => + Right(Await.result(stream.map(_._1).runWith(Sink.seq), 60.seconds)) + case ElasticSuccess(other) => Left(s"unexpected result $other") + case ElasticFailure(error) => Left(error.message) + } + }.toEither.left.map(t => s"${t.getClass.getSimpleName}: ${t.getMessage}").joinRight + + // ------------------------------------------------------------------------------------------- + // The oracle + // ------------------------------------------------------------------------------------------- + + /** An expected value: a number, an instant (a DATE is the instant its day starts at), a boolean, + * a number within a tolerance (a clock read when the statement ran), or NULL. + */ + private sealed trait Expected + private final case class Num(value: Double) extends Expected + private final case class About(value: Double, tolerance: Double) extends Expected + private final case class At(value: Instant) extends Expected + private final case class Bool(value: Boolean) extends Expected + private case object Null extends Expected + + /** A whole number, answered in an integer type (`Integer`, `Long`) -- as a SQL engine answers an + * integer expression -- never as a `Double` or a string. + */ + private final case class Whole(value: Long) extends Expected + + /** A number answered as a NUMBER, never as the string spelling it. */ + private final case class Numeric(value: Double) extends Expected + + /** A number answered in exactly this class, on every row -- what tells a `Double` from an + * `Integer`, which `Num` and `Numeric` accept alike: a `Double` for a fractional type; for a + * whole one the class Elasticsearch's JSON hands it back in, an `Integer` when it fits one and a + * `Long` past it (a BIGINT 3 is read as an `Integer` 3, as `CAST(n AS BIGINT)` itself is). + */ + private final case class Classed(value: Double, cls: Class[_]) extends Expected + + private def doubleOf(value: Double): Expected = Classed(value, classOf[java.lang.Double]) + + private def wholeOf(value: Long): Expected = + Classed( + value.toDouble, + if (value.isValidInt) classOf[java.lang.Integer] else classOf[java.lang.Long] + ) + + /** An instant read from the clock while the statement ran: between `from` and `to`. */ + private final case class Within(from: Instant, to: Instant) extends Expected + + private def start(d: LocalDate): Instant = d.atStartOfDay(ZoneOffset.UTC).toInstant + + private def utcDay(i: Instant): LocalDate = i.atZone(ZoneOffset.UTC).toLocalDate + + private def daysBetween(from: LocalDate, to: LocalDate): Expected = + Num(ChronoUnit.DAYS.between(from, to).toDouble) + + private def fractionalDays(from: Instant, to: Instant): Double = + (to.toEpochMilli - from.toEpochMilli).toDouble / DayMillis + + private def moved(from: Instant, days: Double): Expected = + At(from.plusMillis(Math.round(days * DayMillis))) + + private def expect(value: Option[Expected]): Expected = value.getOrElse(Null) + + /** The UTC day of the statement's clock, read when the test runs. */ + private def today: LocalDate = LocalDate.now(ZoneOffset.UTC) + + /** Elasticsearch wraps every `script_fields` value in a one-element array. */ + private def scalar(v: Any): Any = v match { + case s: Seq[_] => s.headOption.map(scalar).orNull + case c: java.util.Collection[_] => scalar(c.toArray.toSeq) + case other => other + } + + private def asInstant(v: Any): Option[Instant] = + scalar(v) match { + case null => None + case n: java.lang.Number => Some(Instant.ofEpochMilli(Math.round(n.doubleValue()))) + case i: Instant => Some(i) + case z: ZonedDateTime => Some(z.toInstant) + case o: OffsetDateTime => Some(o.toInstant) + case l: LocalDate => Some(start(l)) + case l: LocalDateTime => Some(l.toInstant(ZoneOffset.UTC)) + case d: java.util.Date => Some(d.toInstant) + case s => + val text = s.toString + Try(Instant.parse(text)) + .orElse(Try(ZonedDateTime.parse(text).toInstant)) + .orElse(Try(OffsetDateTime.parse(text).toInstant)) + .orElse(Try(LocalDateTime.parse(text).toInstant(ZoneOffset.UTC))) + .orElse(Try(start(LocalDate.parse(text)))) + .toOption + } + + private def asNumber(v: Any): Option[Double] = + scalar(v) match { + case null => None + case n: java.lang.Number => Some(n.doubleValue()) + case s => Try(s.toString.toDouble).toOption + } + + /** Does a returned value answer the expected one? `None` if it does, else what it answered. */ + private def differs(expected: Expected, actual: Any): Option[String] = + expected match { + case Null => + if (scalar(actual) == null) None else Some(s"$actual instead of NULL") + case Num(want) => + asNumber(actual) match { + case Some(got) if Math.abs(got - want) <= 1e-9 * Math.max(1.0, Math.abs(want)) => None + case _ => Some(s"$actual instead of $want") + } + case About(want, tolerance) => + asNumber(actual) match { + case Some(got) if Math.abs(got - want) <= tolerance => None + case _ => Some(s"$actual instead of about $want") + } + case At(want) => + asInstant(actual) match { + case Some(got) if got == want => None + case _ => Some(s"$actual instead of $want") + } + case Within(from, to) => + asInstant(actual) match { + case Some(got) if !got.isBefore(from) && !got.isAfter(to) => None + case _ => Some(s"$actual instead of an instant between $from and $to") + } + case Bool(want) => + scalar(actual) match { + case b: java.lang.Boolean if b.booleanValue() == want => None + case s: String if Try(s.toBoolean).toOption.contains(want) => None + case other => Some(s"$other instead of $want") + } + case Whole(want) => + scalar(actual) match { + case n @ (_: java.lang.Integer | _: java.lang.Long | _: java.lang.Short | + _: java.lang.Byte) if n.asInstanceOf[Number].longValue() == want => + None + case other => Some(s"${withClass(other)} instead of the whole number $want") + } + case Numeric(want) => + scalar(actual) match { + case n: java.lang.Number + if Math.abs(n.doubleValue() - want) <= 1e-9 * Math.max(1.0, Math.abs(want)) => + None + case other => Some(s"${withClass(other)} instead of the number $want") + } + case Classed(want, cls) => + scalar(actual) match { + case n: java.lang.Number + if n.getClass == cls && + Math.abs(n.doubleValue() - want) <= 1e-9 * Math.max(1.0, Math.abs(want)) => + None + case other => Some(s"${withClass(other)} instead of $want (${cls.getSimpleName})") + } + } + + /** A value with its class: `3.0 (Double)`, `"1" (String)`. */ + private def withClass(value: Any): String = + Option(value).fold("null")(v => s"$v (${v.getClass.getSimpleName})") + + /** A row-level expression and its value for each document. */ + private final case class RowForm(sql: String, value: Row => Expected) + + private def nullIf(a: Option[LocalDate], b: Option[LocalDate]): Option[LocalDate] = + if (a.isDefined && a == b) None else a + + /** GREATEST / LEAST as SQL engines answer them: a NULL argument is skipped, and the answer is + * NULL only when every argument is. + */ + private def latest(values: Option[Instant]*): Option[Instant] = + values.flatten.reduceOption((a, b) => if (a.isAfter(b)) a else b) + + private def earliest(values: Option[Instant]*): Option[Instant] = + values.flatten.reduceOption((a, b) => if (a.isBefore(b)) a else b) + + /** A number cast to a TIMESTAMP is the instant that many milliseconds from the epoch, a + * fractional number truncated as every fractional-to-integral cast is. + */ + private def epochMillis(value: Double): Expected = At(Instant.ofEpochMilli(value.toLong)) + + private val rowForms: Seq[RowForm] = Seq( + RowForm("d - d2", r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(b, a))), + RowForm("d2 - d", r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(a, b))), + RowForm( + "ts - ts2", + r => expect(for (a <- r.ts; b <- r.ts2) yield Num(fractionalDays(b, a))) + ), + RowForm( + "d - ts", + r => expect(for (a <- r.d; b <- r.ts) yield Num(fractionalDays(b, start(a)))) + ), + RowForm( + "ts - d", + r => expect(for (a <- r.ts; b <- r.d) yield Num(fractionalDays(start(b), a))) + ), + RowForm("d + 1", r => expect(r.d.map(a => At(start(a.plusDays(1)))))), + RowForm("d - 1", r => expect(r.d.map(a => At(start(a.minusDays(1)))))), + RowForm("1 + d", r => expect(r.d.map(a => At(start(a.plusDays(1)))))), + RowForm("d + n", r => expect(for (a <- r.d; n <- r.n) yield At(start(a.plusDays(n.toLong))))), + RowForm("n + d", r => expect(for (a <- r.d; n <- r.n) yield At(start(a.plusDays(n.toLong))))), + RowForm("d + 1.5", r => expect(r.d.map(a => moved(start(a), 1.5)))), + RowForm("d - 1.5", r => expect(r.d.map(a => moved(start(a), -1.5)))), + RowForm("d + x", r => expect(for (a <- r.d; x <- r.x) yield moved(start(a), x))), + RowForm("ts + 1", r => expect(r.ts.map(a => moved(a, 1)))), + RowForm("ts - 1.5", r => expect(r.ts.map(a => moved(a, -1.5)))), + RowForm("ts + n", r => expect(for (a <- r.ts; n <- r.n) yield moved(a, n.toDouble))), + RowForm("ts + x", r => expect(for (a <- r.ts; x <- r.x) yield moved(a, x))), + RowForm("d + NULL", _ => Null), + RowForm("NULL - ts", _ => Null), + RowForm( + "CAST('2024-02-15' AS DATE) - d", + r => expect(r.d.map(a => daysBetween(a, LocalDate.parse("2024-02-15")))) + ), + RowForm( + "d + 1 - d2", + r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(b, a.plusDays(1))) + ), + RowForm( + "(ts - ts2) * 24", + r => + expect( + for (a <- r.ts; b <- r.ts2) + yield Num((a.toEpochMilli - b.toEpochMilli) * 24.0 / DayMillis) + ) + ), + // ---- the fix round ---- + // operands that are not bare columns + RowForm( + "COALESCE(d, d2) - d2", + r => expect(for (b <- r.d2; a <- r.d.orElse(r.d2)) yield daysBetween(b, a)) + ), + RowForm( + "NULLIF(d, d2) - d2", + r => expect(for (b <- r.d2; a <- nullIf(r.d, r.d2)) yield daysBetween(b, a)) + ), + RowForm( + "CASE WHEN n > 0 THEN d ELSE d2 END - d", + r => + expect( + for (a <- if (r.n.exists(_ > 0)) r.d else r.d2; b <- r.d) yield daysBetween(b, a) + ) + ), + RowForm( + "CAST(ts AS DATE) - d", + r => expect(for (t <- r.ts; b <- r.d) yield daysBetween(b, utcDay(t))) + ), + // a string literal that is a number counts as that number + RowForm("d + '1'", r => expect(r.d.map(a => At(start(a.plusDays(1)))))), + RowForm("d - '1.5'", r => expect(r.d.map(a => moved(start(a), -1.5)))), + // a typed NULL takes its type, and its value is NULL + RowForm("CAST(NULL AS DATE) - d", _ => Null), + RowForm("d + CAST(NULL AS INT)", _ => Null), + // a cast over arithmetic applies core's cast rule to the RESULT, and keeps NULL a NULL + RowForm( + "(ts - ts2)::BIGINT", + r => expect(for (a <- r.ts; b <- r.ts2) yield Num(fractionalDays(b, a).toLong.toDouble)) + ), + RowForm( + "(n + x)::BIGINT", + r => expect(for (n <- r.n; x <- r.x) yield Num((n + x).toLong.toDouble)) + ), + RowForm("(x * 2)::INT", r => expect(r.x.map(x => Num((x * 2).toInt.toDouble)))), + RowForm( + "(d - d2)::TIMESTAMP", + r => + expect( + for (a <- r.d; b <- r.d2) + yield At(Instant.ofEpochMilli(ChronoUnit.DAYS.between(b, a))) + ) + ), + RowForm( + "(ts - ts2)::TIMESTAMP", + r => + expect( + for (a <- r.ts; b <- r.ts2) yield At(Instant.ofEpochMilli(fractionalDays(b, a).toLong)) + ) + ), + RowForm( + "(d - d2)::DOUBLE", + r => expect(for (a <- r.d; b <- r.d2) yield daysBetween(b, a)) + ), + RowForm("(n + x)::BOOLEAN", r => expect(for (n <- r.n; x <- r.x) yield Bool(n + x != 0))), + // ---- fix round 2 ---- + // GREATEST / LEAST over dates and timestamps: a DATE is its day's start, a NULL is skipped + RowForm("GREATEST(d, d2)", r => expect(latest(r.d.map(start), r.d2.map(start)).map(At))), + RowForm("LEAST(d, ts)", r => expect(earliest(r.d.map(start), r.ts).map(At))), + RowForm("LEAST(ts, ts2)", r => expect(earliest(r.ts, r.ts2).map(At))), + RowForm("GREATEST(d, NULL)", r => expect(r.d.map(a => At(start(a))))), + RowForm( + "GREATEST(d, d2) - d", + r => + expect( + for (a <- r.d; m <- latest(r.d.map(start), r.d2.map(start))) + yield daysBetween(a, utcDay(m)) + ) + ), + // a number cast to a TIMESTAMP: the instant that many milliseconds from the epoch + RowForm("CAST(n AS TIMESTAMP)", r => expect(r.n.map(n => epochMillis(n.toDouble)))), + RowForm("CAST(x AS TIMESTAMP)", r => expect(r.x.map(epochMillis))) + ) + + /** A WHERE predicate and whether each document satisfies it -- `None` (UNKNOWN) keeps none. */ + private final case class WhereForm(sql: String, keeps: Row => Option[Boolean]) + + private def on(s: String): LocalDate = LocalDate.parse(s) + private def at(s: String): Instant = Instant.parse(s) + + private val whereForms: Seq[WhereForm] = Seq( + WhereForm( + "d - d2 = -1", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a) == -1 + ), + WhereForm( + "d2 - d = 1", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(a, b) == 1 + ), + WhereForm( + "d - d2 = 2", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a) == 2 + ), + WhereForm( + "ts - ts2 < -1", + r => for (a <- r.ts; b <- r.ts2) yield fractionalDays(b, a) < -1 + ), + WhereForm( + "d - ts > -0.75", + r => for (a <- r.d; b <- r.ts) yield fractionalDays(b, start(a)) > -0.75 + ), + WhereForm( + "d + 1 - d2 = 0", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a.plusDays(1)) == 0 + ), + WhereForm( + "d - 1 - d2 = 1", + r => for (a <- r.d; b <- r.d2) yield ChronoUnit.DAYS.between(b, a.minusDays(1)) == 1 + ), + WhereForm( + "n + d - d2 = 2", + r => + for (a <- r.d; b <- r.d2; n <- r.n) + yield ChronoUnit.DAYS.between(b, a.plusDays(n.toLong)) == 2 + ), + WhereForm( + "d + 1.5 - ts > 1", + r => + for (a <- r.d; b <- r.ts) + yield (start(a).toEpochMilli + 129600000L - b.toEpochMilli).toDouble / DayMillis > 1 + ), + WhereForm( + "ts + 1 - ts2 > 0", + r => for (a <- r.ts; b <- r.ts2) yield a.toEpochMilli + DayMillis - b.toEpochMilli > 0 + ), + WhereForm( + "ts - 1.5 - ts2 < -1.5", + r => + for (a <- r.ts; b <- r.ts2) + yield (a.toEpochMilli - 129600000L - b.toEpochMilli).toDouble / DayMillis < -1.5 + ), + WhereForm("d + NULL > CAST('2000-01-01' AS DATE)", _ => None), + // ---- refused at PARSE before the fix round, by a type the column did not have yet ---- + WhereForm( + "CURRENT_DATE - d > 30", + r => r.d.map(a => ChronoUnit.DAYS.between(a, today) > 30) + ), + WhereForm("d + 7 > CURRENT_DATE", r => r.d.map(a => a.plusDays(7).isAfter(today))), + WhereForm( + "NOW() - ts > 1", + r => r.ts.map(t => fractionalDays(t, Instant.now()) > 1) + ), + WhereForm( + "CAST(ts AS DATE) - d = 0", + r => for (t <- r.ts; a <- r.d) yield utcDay(t) == a + ), + WhereForm( + "d + 1 = CAST('2024-02-01' AS DATE)", + r => r.d.map(_.plusDays(1) == on("2024-02-01")) + ), + WhereForm("d + 1 > '2024-01-01'", r => r.d.map(_.plusDays(1).isAfter(on("2024-01-01")))), + // ---- fix round 2 ---- + WhereForm( + "GREATEST(d, d2) > CAST('2024-02-01' AS DATE)", + r => latest(r.d.map(start), r.d2.map(start)).map(_.isAfter(start(on("2024-02-01")))) + ), + WhereForm( + "LEAST(ts, ts2) < CAST('2024-02-01T00:00:00Z' AS TIMESTAMP)", + r => earliest(r.ts, r.ts2).map(_.isBefore(at("2024-02-01T00:00:00Z"))) + ), + WhereForm( + "CAST(n AS TIMESTAMP) > CAST('1970-01-01T00:00:00Z' AS TIMESTAMP)", + r => r.n.map(_ > 0) + ) + ) + + /** The aggregates of one group, `None` where the group has no value. */ + private final case class Group( + g: String, + maxD: Option[LocalDate], + minD: Option[LocalDate], + maxD2: Option[LocalDate], + maxTs: Option[Instant], + minTs: Option[Instant], + maxN: Option[Int], + countD: Int, + distinctD: Int, + avgN: Option[Double], + avgX: Option[Double], + avgD: Option[Double] + ) + + private def mean(values: Seq[Double]): Option[Double] = + if (values.isEmpty) None else Some(values.sum / values.size) + + private lazy val groups: Seq[Group] = rows.groupBy(_.g).toSeq.sortBy(_._1).map { case (g, rs) => + val ds = rs.flatMap(_.d) + val tss = rs.flatMap(_.ts) + Group( + g, + ds.reduceOption((a, b) => if (a.isAfter(b)) a else b), + ds.reduceOption((a, b) => if (a.isBefore(b)) a else b), + rs.flatMap(_.d2).reduceOption((a, b) => if (a.isAfter(b)) a else b), + tss.reduceOption((a, b) => if (a.isAfter(b)) a else b), + tss.reduceOption((a, b) => if (a.isBefore(b)) a else b), + rs.flatMap(_.n).reduceOption(_ max _), + ds.size, + ds.distinct.size, + mean(rs.flatMap(_.n).map(_.toDouble)), + mean(rs.flatMap(_.x)), + mean(ds.map(a => start(a).toEpochMilli.toDouble)) + ) + } + + /** A per-group expression and its value for each group. */ + private final case class GroupForm(sql: String, value: Group => Expected) + + private val groupForms: Seq[GroupForm] = Seq( + GroupForm( + "MAX(d) - MIN(d)", + g => expect(for (a <- g.maxD; b <- g.minD) yield daysBetween(b, a)) + ), + GroupForm( + "MIN(d) - MAX(d)", + g => expect(for (a <- g.maxD; b <- g.minD) yield daysBetween(a, b)) + ), + GroupForm( + "MAX(ts) - MIN(ts)", + g => expect(for (a <- g.maxTs; b <- g.minTs) yield Num(fractionalDays(b, a))) + ), + GroupForm( + "MAX(d) - MIN(ts)", + g => expect(for (a <- g.maxD; b <- g.minTs) yield Num(fractionalDays(b, start(a)))) + ), + GroupForm( + "MAX(ts) - MIN(d)", + g => expect(for (a <- g.maxTs; b <- g.minD) yield Num(fractionalDays(start(b), a))) + ), + GroupForm("MAX(d) + 1", g => expect(g.maxD.map(a => At(start(a.plusDays(1)))))), + GroupForm("MAX(d) - 1", g => expect(g.maxD.map(a => At(start(a.minusDays(1)))))), + GroupForm("1 + MAX(d)", g => expect(g.maxD.map(a => At(start(a.plusDays(1)))))), + GroupForm("MAX(d) + 1.5", g => expect(g.maxD.map(a => moved(start(a), 1.5)))), + GroupForm("MAX(ts) + 1", g => expect(g.maxTs.map(a => moved(a, 1)))), + GroupForm("MAX(ts) - 1.5", g => expect(g.maxTs.map(a => moved(a, -1.5)))), + GroupForm( + "MAX(d) + MAX(n)", + g => expect(for (a <- g.maxD; n <- g.maxN) yield At(start(a.plusDays(n.toLong)))) + ), + GroupForm("MAX(d) - NULL", _ => Null), + GroupForm( + "MAX(d) - CAST('2024-01-01' AS DATE)", + g => expect(g.maxD.map(a => daysBetween(LocalDate.parse("2024-01-01"), a))) + ), + // ---- the fix round ---- + // an aggregate is the metric it produces: COUNT counts, AVG averages + GroupForm("COUNT(d) - 1", g => Num(g.countD - 1.0)), + GroupForm("COUNT(DISTINCT d) * 2", g => Num(g.distinctD * 2.0)), + GroupForm("AVG(d) - 1", g => expect(g.avgD.map(a => Num(a - 1)))), + GroupForm( + "MAX(d) - AVG(n)", + g => expect(for (a <- g.maxD; n <- g.avgN) yield moved(start(a), -n)) + ), + // the request clock in a per-group calculation + GroupForm( + "CURRENT_DATE - MAX(d)", + g => expect(g.maxD.map(a => daysBetween(a, today))) + ), + GroupForm( + "DATEDIFF(CURRENT_DATE, MAX(d))", + g => expect(g.maxD.map(a => daysBetween(a, today))) + ), + GroupForm( + "NOW() - MAX(ts)", + g => expect(g.maxTs.map(t => About(fractionalDays(t, Instant.now()), 1.0 / 24))) + ), + // a cast over a per-group calculation + GroupForm( + "(MAX(n) + AVG(x))::BIGINT", + g => expect(for (n <- g.maxN; x <- g.avgX) yield Num((n + x).toLong.toDouble)) + ), + // ---- fix round 2: GREATEST / LEAST over aggregates, computed per group ---- + // ⚠️ A SELECT's per-group calculation is NULL in a group missing one of its metrics: + // Elasticsearch's bucket_script does not run there (its gap policy skips the group), as for + // every per-group calculation. A HAVING reads every aggregate and skips a NULL one (below). + GroupForm( + "GREATEST(MAX(d), MAX(d2))", + g => expect(for (a <- g.maxD; b <- g.maxD2) yield At(start(if (a.isAfter(b)) a else b))) + ), + GroupForm( + "LEAST(MAX(ts), MAX(d))", + g => + expect(for (t <- g.maxTs; a <- g.maxD; m <- earliest(Some(t), Some(start(a)))) yield At(m)) + ), + GroupForm( + "GREATEST(MAX(d), MAX(d2)) - MIN(d)", + g => + expect( + for (a <- g.maxD; b <- g.maxD2; m <- g.minD) + yield daysBetween(m, if (a.isAfter(b)) a else b) + ) + ), + GroupForm("GREATEST(MAX(n), 2)", g => expect(g.maxN.map(n => Num(Math.max(n, 2).toDouble)))) + ) + + /** A HAVING over a per-group calculation and whether each group is kept. */ + private final case class HavingForm(sql: String, keeps: Group => Option[Boolean]) + + private val havingForms: Seq[HavingForm] = Seq( + HavingForm( + s"SELECT g, MAX(d) - MIN(d) AS x FROM $table GROUP BY g HAVING x > 10", + g => for (a <- g.maxD; b <- g.minD) yield ChronoUnit.DAYS.between(b, a) > 10 + ), + HavingForm( + s"SELECT g, MAX(d) - MIN(d) AS x FROM $table GROUP BY g HAVING x = 0", + g => for (a <- g.maxD; b <- g.minD) yield ChronoUnit.DAYS.between(b, a) == 0 + ), + HavingForm( + s"SELECT g, MAX(ts) - MIN(ts) AS x FROM $table GROUP BY g HAVING x > 0.5", + g => for (a <- g.maxTs; b <- g.minTs) yield fractionalDays(b, a) > 0.5 + ), + HavingForm( + s"SELECT g, MAX(d) - MIN(ts) AS x FROM $table GROUP BY g HAVING x < 0", + g => for (a <- g.maxD; b <- g.minTs) yield fractionalDays(b, start(a)) < 0 + ), + HavingForm( + s"SELECT g, MAX(d) + 1 AS x FROM $table GROUP BY g HAVING x > MAX(d2)", + g => for (a <- g.maxD; b <- g.maxD2) yield a.plusDays(1).isAfter(b) + ), + HavingForm( + s"SELECT g, MAX(d) + 1.5 AS x FROM $table GROUP BY g HAVING x > MAX(ts)", + g => for (a <- g.maxD; b <- g.maxTs) yield start(a).plusMillis(129600000L).isAfter(b) + ), + HavingForm( + s"SELECT g, MAX(ts) - 1 AS x FROM $table GROUP BY g HAVING x < MIN(ts)", + g => for (a <- g.maxTs; b <- g.minTs) yield a.minusMillis(DayMillis).isBefore(b) + ), + HavingForm( + s"SELECT g FROM $table GROUP BY g HAVING MAX(d) > CAST('2024-03-01' AS DATE) - 30", + g => g.maxD.map(_.isAfter(on("2024-01-31"))) + ), + // refused at PARSE before the fix round + HavingForm( + s"SELECT g, MAX(d) + 1 AS x FROM $table GROUP BY g HAVING x > CAST('2024-01-01' AS DATE)", + g => g.maxD.map(_.plusDays(1).isAfter(on("2024-01-01"))) + ), + HavingForm( + s"SELECT g, COUNT(d) - 1 AS x FROM $table GROUP BY g HAVING x > 0", + g => Some(g.countD - 1 > 0) + ), + // ---- fix round 2: GREATEST over aggregates. Inline, the group filter reads every aggregate + // and SKIPS a NULL one (g2 has a `d` and no `d2`); through its SELECT alias it reads the + // per-group calculation, NULL in a group missing one of its metrics ---- + HavingForm( + s"SELECT g FROM $table GROUP BY g HAVING GREATEST(MAX(d), MAX(d2)) > CAST('2023-12-01' AS DATE)", + g => latest(g.maxD.map(start), g.maxD2.map(start)).map(_.isAfter(start(on("2023-12-01")))) + ), + HavingForm( + s"SELECT g, GREATEST(MAX(d), MAX(d2)) AS x FROM $table GROUP BY g HAVING x > CAST('2023-12-01' AS DATE)", + g => + for (a <- g.maxD; b <- g.maxD2) yield (if (a.isAfter(b)) a else b).isAfter(on("2023-12-01")) + ) + ) + + /** Run every statement before asserting anything, so a failure names every one that differs. */ + private def verdict(label: String, outcomes: Seq[(String, Either[String, Seq[String]])]): Unit = { + val wrong = outcomes.collect { + case (sql, Right(diffs)) if diffs.nonEmpty => s"[$sql] ${diffs.mkString("; ")}" + } + val errors = outcomes.collect { case (sql, Left(error)) => s"[$sql] $error" } + info(s"$label: ${outcomes.size} statements; wrong: ${wrong.size}; errors: ${errors.size}") + (wrong ++ errors).foreach(info(_)) + withClue( + s"${wrong.size} wrong, ${errors.size} errors:\n${(wrong ++ errors).take(30).mkString("\n")}\n" + ) { + wrong shouldBe empty + errors shouldBe empty + } + } + + private def byKey(rs: Seq[ListMap[String, Any]], key: String): Map[String, Any] = + rs.map(r => r.getOrElse(key, "?").toString -> r.getOrElse("x", null)).toMap + + // ------------------------------------------------------------------------------------------- + // The venues + // ------------------------------------------------------------------------------------------- + + private def rowVerdicts( + forms: Seq[RowForm], + from: String, + population: Seq[Row] + ): Seq[(String, Either[String, Seq[String]])] = + forms.map { f => + val sql = s"SELECT id, ${f.sql} AS x FROM $from" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + population.flatMap { r => + differs(f.value(r), got.getOrElse(r.id, null)).map(d => s"${r.id}: $d") + } ++ (if (got.size == population.size) Nil else Seq(s"${got.size} rows")) + } + } + + "date arithmetic in a row SELECT" should "answer the java.time oracle on every document" in { + rowForms.flatMap(f => rows.map(f.value)).distinct.size should be > 10 + verdict("row SELECT", rowVerdicts(rowForms, table, rows)) + } + + private def whereVerdicts( + forms: Seq[WhereForm], + from: String = table + ): Seq[(String, Either[String, Seq[String]])] = + forms.map { f => + val sql = s"SELECT id FROM $from WHERE ${f.sql}" + sql -> gatewayRows(sql).map { rs => + val kept = rs.map(_.getOrElse("id", "?").toString).toSet + val want = rows.filter(r => f.keeps(r).contains(true)).map(_.id).toSet + if (kept == want) Nil + else + Seq( + s"kept ${kept.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}" + ) + } + } + + "date arithmetic in a WHERE" should "keep exactly the documents the java.time oracle keeps" in { + // the predicates keep some documents and drop others + whereForms.exists(f => rows.exists(r => f.keeps(r).contains(true))) shouldBe true + whereForms.exists(f => rows.exists(r => !f.keeps(r).contains(true))) shouldBe true + verdict("WHERE", whereVerdicts(whereForms)) + } + + /** A date column compared with date arithmetic -- the filter `d > CURRENT_DATE - 7` is written + * for -- and the neighbouring comparisons a fix for it must keep working. On Elasticsearch 6.8 a + * `date` doc value is a `JodaCompatibleZonedDateTime`, which `isBefore` / `isAfter` / `isEqual` + * against a genuine `ZonedDateTime` used to fail with `Class.cast`. + * + * ⚠️ A column compared with ANOTHER column (`d2 > d`) is not here: on Elasticsearch 6.8 it is + * sent as a `range` bounded by the other column's NAME, outside this suite's rules. + */ + private val columnForms: Seq[WhereForm] = Seq( + WhereForm("d < CAST('2024-01-07' AS DATE) - 7", r => r.d.map(_.isBefore(on("2023-12-31")))), + WhereForm( + "d >= CAST('2024-03-08' AS DATE) - 7", + r => r.d.map(a => !a.isBefore(on("2024-03-01"))) + ), + WhereForm( + "ts > CAST('2024-03-01T00:00:00Z' AS TIMESTAMP) - 1.5", + r => r.ts.map(_.isAfter(at("2024-02-28T12:00:00Z"))) + ), + WhereForm("d2 = d + 1", r => for (a <- r.d; b <- r.d2) yield b == a.plusDays(1)), + WhereForm("d > d2 - 1", r => for (a <- r.d; b <- r.d2) yield a.isAfter(b.minusDays(1))), + WhereForm( + "DATE_ADD(d, INTERVAL 1 DAY) = d2", + r => for (a <- r.d; b <- r.d2) yield a.plusDays(1) == b + ), + WhereForm("CAST(ts AS DATE) = d", r => for (t <- r.ts; a <- r.d) yield utcDay(t) == a), + WhereForm( + "COALESCE(d, d2) > CAST('2024-01-01' AS DATE)", + r => r.d.orElse(r.d2).map(_.isAfter(on("2024-01-01"))) + ), + WhereForm( + "ts < ts2 + 0.5", + r => for (a <- r.ts; b <- r.ts2) yield a.isBefore(b.plusMillis(DayMillis / 2)) + ), + WhereForm( + "d >= CAST(ts AS DATE) - 1", + r => for (a <- r.d; t <- r.ts) yield !a.isBefore(utcDay(t).minusDays(1)) + ), + WhereForm("d + 1 > d2", r => for (a <- r.d; b <- r.d2) yield a.plusDays(1).isAfter(b)), + WhereForm("d - 1 = d2", r => for (a <- r.d; b <- r.d2) yield a.minusDays(1) == b), + WhereForm("CURRENT_DATE > d", r => r.d.map(today.isAfter(_))), + // ---- fix round 2: GREATEST / LEAST over dates against a date column ---- + WhereForm( + "GREATEST(d, d2) > d", + r => for (a <- r.d; m <- latest(r.d.map(start), r.d2.map(start))) yield m.isAfter(start(a)) + ), + WhereForm( + "LEAST(d, ts) < ts", + r => for (t <- r.ts; m <- earliest(r.d.map(start), r.ts)) yield m.isBefore(t) + ), + // the comparisons that already worked, on every major + WhereForm("d = CAST('2024-01-31' AS DATE)", r => r.d.map(_ == on("2024-01-31"))), + WhereForm("d >= '2024-01-01'", r => r.d.map(a => !a.isBefore(on("2024-01-01")))), + WhereForm( + "CAST(d AS TIMESTAMP) < ts", + r => for (a <- r.d; t <- r.ts) yield start(a).isBefore(t) + ), + WhereForm( + "n + d = CAST('2024-02-03' AS DATE)", + r => for (a <- r.d; n <- r.n) yield a.plusDays(n.toLong) == on("2024-02-03") + ), + WhereForm("d > CAST('2024-01-01' AS DATE)", r => r.d.map(_.isAfter(on("2024-01-01")))), + WhereForm( + "ts > CAST('2024-01-31T00:00:00Z' AS TIMESTAMP)", + r => r.ts.map(_.isAfter(at("2024-01-31T00:00:00Z"))) + ), + WhereForm("d < CURRENT_DATE", r => r.d.map(_.isBefore(today))), + WhereForm( + "CAST(d AS DATE) = CAST('2024-01-31' AS DATE)", + r => r.d.map(_ == on("2024-01-31")) + ), + WhereForm( + "d BETWEEN CAST('2024-01-01' AS DATE) AND CAST('2024-12-31' AS DATE)", + r => r.d.map(a => !a.isBefore(on("2024-01-01")) && !a.isAfter(on("2024-12-31"))) + ), + WhereForm( + "d > CAST('2024-03-08' AS DATE) - INTERVAL 7 DAY", + r => r.d.map(_.isAfter(on("2024-03-01"))) + ) + ) + + "a date column compared with date arithmetic" should "keep exactly the documents the java.time oracle keeps" in { + columnForms.exists(f => rows.exists(r => f.keeps(r).contains(true))) shouldBe true + columnForms.exists(f => rows.exists(r => !f.keeps(r).contains(true))) shouldBe true + verdict("column vs date arithmetic", whereVerdicts(columnForms)) + } + + private def groupVerdicts( + forms: Seq[GroupForm], + from: String = table + ): Seq[(String, Either[String, Seq[String]])] = + forms.map { f => + val sql = s"SELECT g, ${f.sql} AS x FROM $from GROUP BY g" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "g") + groups.flatMap { g => + differs(f.value(g), got.getOrElse(g.g, null)).map(d => s"${g.g}: $d") + } ++ (if (got.size == groups.size) Nil else Seq(s"${got.size} groups")) + } + } + + "date arithmetic per group" should "answer the java.time oracle in every group, NULL where an operand has no value" in { + // a group with two dates, groups with one, and a non-empty group with none + groups.map(_.g) shouldBe Seq("g1", "g2", "g3", "g4", "g5", "g6") + groups.find(_.g == "g3").flatMap(_.maxD) shouldBe None + verdict("per-group SELECT", groupVerdicts(groupForms)) + } + + "a HAVING over date arithmetic" should "keep exactly the groups the java.time oracle keeps" in { + havingForms.exists(f => groups.exists(g => f.keeps(g).contains(true))) shouldBe true + havingForms.exists(f => groups.exists(g => !f.keeps(g).contains(true))) shouldBe true + verdict( + "HAVING", + havingForms.map { f => + f.sql -> gatewayRows(f.sql).map { rs => + val kept = rs.map(_.getOrElse("g", "?").toString).toSet + val want = groups.filter(g => f.keeps(g).contains(true)).map(_.g).toSet + if (kept == want) Nil + else + Seq( + s"kept ${kept.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}" + ) + } + } + ) + } + + "a DATE aggregate over date-time strings" should + "compare as its UTC day's start per group, in GREATEST / LEAST as in date arithmetic" in { + // Elasticsearch accepts a date-time into a `date` field, and MAX / MIN of it keep that time: + // per group, a DATE operand is still its UTC day -- the oracle is the one of `table`, whose + // `d` is the same days without a time + run(s"CREATE TABLE $dateTimeTable ($columns, PRIMARY KEY (id))") + bulk( + dateTimeTable, + rows.map(r => json(r.copy(dText = r.d.map(day => s"${day}T22:30:00Z")))).toList + ) + val forms = groupForms.filter(f => + Set( + "GREATEST(MAX(d), MAX(d2))", + "LEAST(MAX(ts), MAX(d))", + "GREATEST(MAX(d), MAX(d2)) - MIN(d)", + "MAX(d) + 1", + "MAX(d) - MIN(ts)" + ).contains(f.sql) + ) + forms.size shouldBe 5 + // a group filter reads every aggregate, NULL ones skipped: g1's LEAST is its day's start, + // 2024-03-01T00:00, kept; read with its time, it would be 06:00, and g1 would be dropped + val threshold = Instant.parse("2024-03-01T03:00:00Z") + val having = + s"SELECT g FROM $dateTimeTable GROUP BY g " + + s"HAVING LEAST(MAX(ts), MAX(d)) < CAST('$threshold' AS TIMESTAMP)" + val want = groups + .filter(g => earliest(g.maxTs, g.maxD.map(start)).exists(_.isBefore(threshold))) + .map(_.g) + .toSet + want should contain("g1") + verdict( + s"FROM $dateTimeTable", + groupVerdicts(forms, dateTimeTable) :+ (having -> gatewayRows(having).map { rs => + val kept = rs.map(_.getOrElse("g", "?").toString).toSet + if (kept == want) Nil + else + Seq( + s"kept ${kept.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}" + ) + }) + ) + } + + "a computed column over date arithmetic" should "STORE the java.time oracle's value" in { + val forms: Map[String, RowForm] = rowForms.map(f => f.sql -> f).toMap + val stored = rows ++ computedOnlyRows + val sql = s"SELECT id, ${computed.map(_._1).mkString(", ")} FROM $computedTable" + verdict( + "SCRIPT AS", + Seq(sql -> gatewayRows(sql).map { rs => + val byId = rs.map(r => r.getOrElse("id", "?").toString -> r).toMap + (if (byId.size == stored.size) Nil else Seq(s"${byId.size} rows")) ++ + (for { + (c, _, expression) <- computed + r <- stored + } yield { + val want = + forms.get(expression).map(_.value(r)).getOrElse(fail(s"no oracle for $expression")) + differs(want, byId.get(r.id).flatMap(_.get(c)).orNull).map(d => + s"$c [$expression] ${r.id}: $d" + ) + }).flatten + }) + ) + } + + /** The type DESCRIBE reports for `field` of `index`. */ + private def described(index: String, field: String): Either[String, Option[String]] = + gatewayRows(s"DESCRIBE TABLE $index").map( + _.find(_.get("Field").contains(field)).flatMap(_.get("Type")).map(_.toString) + ) + + private val loaded = Seq( + ("n1", "a", "2024-01-31T10:30:00Z", "2024-02-01T09:15:00Z"), + ("n2", "a", "2024-01-31T23:00:00Z", "2024-02-01T01:00:00Z"), + ("n3", "b", "1969-12-31T23:00:00Z", "1970-01-01T01:00:00Z") + ) + + /** The undeclared dates keep their time: `ts + 1`, `ts2 - ts`, `ts - 1.5` over `index`. */ + private def undeclaredVerdicts(index: String): Seq[(String, Either[String, Seq[String]])] = { + def ts(s: String) = Instant.parse(s) + Seq( + s"SELECT id, ts + 1 AS x FROM $index" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), 1) + }, + s"SELECT id, ts2 - ts AS x FROM $index" -> loaded.map { case (id, _, t, t2) => + id -> Num(fractionalDays(ts(t), ts(t2))) + }, + s"SELECT id, ts - 1.5 AS x FROM $index" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), -1.5) + } + ).map { case (sql, expected) => + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + expected.flatMap { case (id, want) => + differs(want, got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + } + } + + "an index created by a bulk load, with no declaration" should + "hold TIMESTAMPs: its dates keep their time, and DESCRIBE reports them so" in { + bulk( + rawIndex, + loaded.map { case (id, g, ts, ts2) => + s"""{"id":"$id","g":"$g","ts":"$ts","ts2":"$ts2"}""" + }.toList + ) + // the lead's ruling of 2026-10-05: a `date` field with no elasticsql declaration is a + // TIMESTAMP everywhere, DESCRIBE included + described(rawIndex, "ts") shouldBe Right(Some("TIMESTAMP")) + def ts(s: String) = Instant.parse(s) + verdict( + "undeclared date", + Seq( + s"SELECT id, ts + 1 AS x FROM $rawIndex" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), 1) + }, + s"SELECT id, ts2 - ts AS x FROM $rawIndex" -> loaded.map { case (id, _, t, t2) => + id -> Num(fractionalDays(ts(t), ts(t2))) + }, + s"SELECT id, ts - 1.5 AS x FROM $rawIndex" -> loaded.map { case (id, _, t, _) => + id -> moved(ts(t), -1.5) + } + ).map { case (sql, expected) => + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + expected.flatMap { case (id, want) => + differs(want, got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + } :+ { + val sql = + s"SELECT g.keyword AS g, MAX(ts2) - MIN(ts) AS x FROM $rawIndex GROUP BY g.keyword" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "g") + loaded.groupBy(_._2).toSeq.flatMap { case (g, members) => + val want = Num( + fractionalDays(members.map(m => ts(m._3)).min, members.map(m => ts(m._4)).max) + ) + differs(want, got.getOrElse(g, null)).map(d => s"$g: $d") + } + } + } + ) + } + + it should "stay a TIMESTAMP once an elasticsql ALTER TABLE declares the table, cache refreshed" in { + bulk( + alteredRawIndex, + loaded.map { case (id, g, ts, ts2) => + s"""{"id":"$id","g":"$g","ts":"$ts","ts2":"$ts2"}""" + }.toList + ) + verdict("before the ALTER", undeclaredVerdicts(alteredRawIndex)) + // an unrelated ALTER writes the table's `_meta`, every column of it: the undeclared dates are + // declared as what they are, TIMESTAMPs -- never turned into DATEs + run(s"ALTER TABLE $alteredRawIndex ADD COLUMN note KEYWORD") + client.invalidateAllSchemas() + described(alteredRawIndex, "ts") shouldBe Right(Some("TIMESTAMP")) + described(alteredRawIndex, "note") shouldBe Right(Some("KEYWORD")) + verdict("after the ALTER and a cache refresh", undeclaredVerdicts(alteredRawIndex)) + } + + "a date field a bulk load adds to a table elasticsql created" should + "be a TIMESTAMP, while the declared DATE stays a DATE" in { + run(s"CREATE TABLE $grownTable (id KEYWORD, d DATE, PRIMARY KEY (id))") + bulk( + grownTable, + List("""{"id":"w1","d":"2024-01-31","seen":"2024-01-31T10:30:00Z"}""") + ) + client.invalidateAllSchemas() + described(grownTable, "d") shouldBe Right(Some("DATE")) + described(grownTable, "seen") shouldBe Right(Some("TIMESTAMP")) + verdict( + "a dynamic date field", + Seq(s"SELECT id, seen + 1 AS x, d + 1 AS y FROM $grownTable" -> { + val sql = s"SELECT id, seen + 1 AS x, d + 1 AS y FROM $grownTable" + gatewayRows(sql).map { rs => + val got = rs.headOption.getOrElse(ListMap.empty[String, Any]) + differs(At(Instant.parse("2024-02-01T10:30:00Z")), got.getOrElse("x", null)).toSeq ++ + differs(At(start(on("2024-02-01"))), got.getOrElse("y", null)).toSeq + } + }) + ) + } + + /** The comparison rule's fixture: base's census rows (r1 to r4), and r5, whose `n` = 10 and `s` = + * "12" tell a number read as a NUMBER from one compared as TEXT ("10" < "2"). + */ + private val comparisonDocs = List( + """{"id":"r1","g":"g1","d":"2024-01-31","d2":"2024-02-01","ts":"2024-01-31T10:30:00Z","n":3,"s":"a"}""", + """{"id":"r2","g":"g1","d":"2024-03-01","d2":"2024-02-28","ts":"2024-03-01T06:00:00Z","n":5,"s":"5"}""", + """{"id":"r3","g":"g2","d":"2023-12-25","ts":"2023-12-25T23:59:59Z","n":-2,"s":"3"}""", + """{"id":"r4","g":"g3","s":"b"}""", + """{"id":"r5","g":"g3","d2":"2024-01-15","n":10,"s":"12"}""" + ) + + /** `SELECT id, AS v` over [[comparisonTable]], against the expected value of each + * row -- PostgreSQL 16's answer over the same rows, which for r1 to r4 is also base's. + */ + private def comparisonVerdict( + expression: String, + expected: Map[String, Expected], + from: String = comparisonTable + ): (String, Either[String, Seq[String]]) = { + val sql = s"SELECT id, $expression AS x FROM $from" + sql -> gatewayRows(sql).map { rs => + val got = byKey(rs, "id") + expected.toSeq.sortBy(_._1).flatMap { case (id, want) => + differs(want, got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + } + + private def kept(sql: String, want: Set[String]): (String, Either[String, Seq[String]]) = + sql -> gatewayRows(sql).map { rs => + val got = rs.map(r => r.getOrElse("id", r.getOrElse("g", "?")).toString).toSet + if (got == want) Nil + else + Seq(s"kept ${got.toSeq.sorted.mkString(",")}, expected ${want.toSeq.sorted.mkString(",")}") + } + + private def flags(ones: String*): Map[String, Expected] = + Seq("r1", "r2", "r3", "r4", "r5").map(id => id -> Num(if (ones.contains(id)) 1 else 0)).toMap + + "the ONE comparison rule" should + "read a string literal by what it is compared with, in every clause, as PostgreSQL does" in { + run( + s"CREATE TABLE $comparisonTable (id KEYWORD, g KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, " + + "n INT, s KEYWORD, PRIMARY KEY (id))" + ) + bulk(comparisonTable, comparisonDocs) + verdict( + "comparison rule", + Seq( + // the review's statements, accurate on base and refused by round 2: base's values on r1 + // to r4, and r5 read as a number + comparisonVerdict("CASE WHEN COALESCE(n, 0) = '3' THEN 1 ELSE 0 END", flags("r1")), + comparisonVerdict("CASE WHEN n + 1 = '4' THEN 1 ELSE 0 END", flags("r1")), + comparisonVerdict( + "CASE WHEN LENGTH(s) = '1' THEN 1 ELSE 0 END", + flags("r1", "r2", "r3", "r4") + ), + comparisonVerdict( + "CASE WHEN COALESCE(n, 0) > '2' THEN 1 ELSE 0 END", + flags("r1", "r2", "r5") + ), + comparisonVerdict("CASE WHEN NULLIF(n, 0) = '3' THEN 1 ELSE 0 END", flags("r1")), + comparisonVerdict("CASE n WHEN '3' THEN 1 ELSE 0 END", flags("r1")), + kept( + s"SELECT id FROM $comparisonTable WHERE COALESCE(d, d2) BETWEEN '2024-01-01' AND '2024-02-01'", + Set("r1", "r5") + ), + // main refused this one at parse + kept(s"SELECT id FROM $comparisonTable WHERE COALESCE(n, 0) = '3'", Set("r1")), + kept( + s"SELECT id FROM $comparisonTable WHERE COALESCE(d, d2) IN ('2024-01-31', '2024-01-15')", + Set("r1", "r5") + ), + kept( + s"SELECT g, MAX(n) AS m FROM $comparisonTable GROUP BY g HAVING MAX(n) > '4'", + Set("g1", "g3") + ), + // a DATE against a TIMESTAMP is comparable, judged on the declared types + comparisonVerdict( + "CASE d WHEN CAST('2024-01-31' AS DATE) THEN 1 ELSE 0 END", + flags("r1") + ), + comparisonVerdict( + "NULLIF(COALESCE(d, d2), CAST('2024-01-31' AS DATE))", + Map( + "r1" -> Null, + "r2" -> At(start(on("2024-03-01"))), + "r3" -> At(start(on("2023-12-25"))), + "r4" -> Null, + "r5" -> At(start(on("2024-01-15"))) + ) + ), + comparisonVerdict( + "NULLIF(CAST(d AS DATE), CAST(ts AS TIMESTAMP))", + Map( + "r1" -> At(start(on("2024-01-31"))), + "r2" -> At(start(on("2024-03-01"))), + "r3" -> At(start(on("2023-12-25"))), + "r4" -> Null, + "r5" -> Null + ) + ), + // the arithmetic reads it too + comparisonVerdict( + "d - '2024-01-31'", + Map("r1" -> Num(0), "r2" -> Num(30), "r3" -> Num(-37), "r4" -> Null, "r5" -> Null) + ), + comparisonVerdict( + "n + '1'", + Map("r1" -> Num(4), "r2" -> Num(6), "r3" -> Num(-1), "r4" -> Null, "r5" -> Num(11)) + ), + // a literal that spells nothing the operand reads stays refused by name + s"SELECT id, CASE WHEN COALESCE(n, 0) = 'a' THEN 1 ELSE 0 END AS v FROM $comparisonTable" -> + (Try( + Await.result( + client.run( + s"SELECT id, CASE WHEN COALESCE(n, 0) = 'a' THEN 1 ELSE 0 END AS v FROM $comparisonTable" + ), + 60.seconds + ) + ).toEither match { + case Right(ElasticFailure(error)) + if error.message.contains("'BIGINT' is not compatible with 'VARCHAR'") => + Right(Nil) + case other => Right(Seq(s"answered $other instead of the refusal by name")) + }) + ) + ) + } + + /** The documents of [[numberTable]]: an INT, a BIGINT past INT's range, a DOUBLE with a fraction, + * and a row holding none of them. + */ + private val numberDocs = List( + """{"id":"r1","n":1,"big":1,"x":1.5}""", + """{"id":"r2","n":2,"big":10000000000,"x":2.0}""", + """{"id":"r3","n":-3,"big":-3,"x":-1.25}""", + """{"id":"r4"}""" + ) + + "a quoted number" should + "be a value of the other operand's type, as PostgreSQL reads it: any number beside a DOUBLE, a whole one beside an integer" in { + run(s"CREATE TABLE $numberTable (id KEYWORD, n INT, big BIGINT, x DOUBLE, PRIMARY KEY (id))") + bulk(numberTable, numberDocs) + def values(r1: Expected, r2: Expected, r3: Expected, r4: Expected): Map[String, Expected] = + Map("r1" -> r1, "r2" -> r2, "r3" -> r3, "r4" -> r4) + def refused(sql: String, refusal: String): (String, Either[String, Seq[String]]) = + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) if error.message.contains(refusal) => Right(Nil) + case Right(ElasticFailure(error)) => Right(Seq(s"refused as [${error.message}]")) + case Right(other) => Right(Seq(s"answered $other instead of the refusal by name")) + case Left(t) => Left(t.toString) + }) + verdict( + "quoted numbers", + Seq( + // beside a DOUBLE, any number: x = 1.5, 2.0, -1.25 and NULL + comparisonVerdict( + "CASE WHEN COALESCE(x, 0) = '1.5' THEN 1 ELSE 0 END", + values(Num(1), Num(0), Num(0), Num(0)), + numberTable + ), + comparisonVerdict( + "CASE WHEN COALESCE(x, 0) = '2' THEN 1 ELSE 0 END", + values(Num(0), Num(1), Num(0), Num(0)), + numberTable + ), + comparisonVerdict( + "CASE x WHEN '1.5' THEN 1 ELSE 0 END", + values(Num(1), Num(0), Num(0), Num(0)), + numberTable + ), + comparisonVerdict( + "NULLIF(x, '1.5')", + values(Null, Num(2.0), Num(-1.25), Null), + numberTable + ), + comparisonVerdict("x + '1.5'", values(Num(3.0), Num(3.5), Num(0.25), Null), numberTable), + comparisonVerdict( + "x * '2.5'", + values(Num(3.75), Num(5.0), Num(-3.125), Null), + numberTable + ), + kept(s"SELECT id FROM $numberTable WHERE COALESCE(x, 0) IN ('1.5', '3')", Set("r1")), + kept( + s"SELECT id FROM $numberTable WHERE COALESCE(x, 0) BETWEEN '1.0' AND '1.5'", + Set("r1") + ), + kept(s"SELECT id FROM $numberTable WHERE COALESCE(x, 0) > '1.75'", Set("r2")), + // beside an integer, a whole number is one -- past INT's range too, beside a BIGINT + comparisonVerdict( + "CASE WHEN COALESCE(big, 0) = '10000000000' THEN 1 ELSE 0 END", + values(Num(0), Num(1), Num(0), Num(0)), + numberTable + ), + comparisonVerdict("n + '1'", values(Num(2), Num(3), Num(-2), Null), numberTable), + // beside an integer, '1.5' and '1.0' are no integer: refused by name, as PostgreSQL 16 + // refuses them (`invalid input syntax for type integer`) + refused( + s"SELECT id, CASE WHEN COALESCE(n, 0) = '1.5' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id, CASE WHEN NULLIF(n, 0) = '1.0' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'INT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id, CASE WHEN COALESCE(big, 0) = '1.5' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id, CASE WHEN COALESCE(big, 0) = '1.0' THEN 1 ELSE 0 END AS x FROM $numberTable", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id FROM $numberTable WHERE COALESCE(n, 0) IN ('1', '1.5')", + "'BIGINT' is not compatible with 'VARCHAR'" + ), + refused( + s"SELECT id FROM $numberTable WHERE COALESCE(big, 0) BETWEEN '1' AND '1.0'", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ), + refused(s"SELECT id, NULLIF(n, '1.5') AS x FROM $numberTable", "compares INT with VARCHAR"), + refused( + s"SELECT id, CASE big WHEN '1.0' THEN 1 ELSE 0 END AS x FROM $numberTable", + "CASE WHEN conditions must be of the same type as the expression" + ), + refused( + s"SELECT id, n + '1.5' AS x FROM $numberTable", + "output 'INT' is not compatible with input 'VARCHAR'" + ), + refused( + s"SELECT id, big - '1.0' AS x FROM $numberTable", + "output 'BIGINT' is not compatible with input 'VARCHAR'" + ) + ) + ) + } + + "DATE_TRUNC(…, DAY) compared with a date" should "run: a day truncation of a date is the date" in { + verdict( + "DATE_TRUNC", + Seq( + kept( + s"SELECT id FROM $comparisonTable WHERE DATE_TRUNC(ts, DAY) = '2024-01-31'", + Set("r1") + ), + kept( + s"SELECT id FROM $comparisonTable WHERE DATE_TRUNC(ts, DAY) > '2024-01-01'", + Set("r1", "r2") + ), + kept( + s"SELECT id FROM $comparisonTable WHERE DATE_TRUNC(ts, DAY) = CAST('2024-01-31' AS DATE)", + Set("r1") + ), + comparisonVerdict( + "CASE WHEN DATE_TRUNC(ts, DAY) = '2024-01-31' THEN 1 ELSE 0 END", + flags("r1") + ) + ) + ) + } + + /** The STORED value of each computed column of `from`, for each document. */ + private def storedVerdict( + from: String, + expected: Seq[(String, Map[String, Expected])] + ): (String, Either[String, Seq[String]]) = { + val sql = s"SELECT id, ${expected.map(_._1).mkString(", ")} FROM $from" + val ids = expected.flatMap(_._2.keys).toSet + sql -> gatewayRows(sql).map { rs => + val byId = rs.map(r => r.getOrElse("id", "?").toString -> r).toMap + (if (byId.keySet == ids) Nil else Seq(s"rows ${byId.keySet.toSeq.sorted.mkString(",")}")) ++ + (for { + (column, values) <- expected + (id, want) <- values.toSeq.sortBy(_._1) + diff <- differs(want, byId.get(id).flatMap(_.get(column)).orNull) + } yield s"$column $id: $diff") + } + } + + /** The documents of [[nullifTable]], PostgreSQL 16's fixture for the answers below: a DATE that + * is the literals' day and one that is not, TIMESTAMPs at that day's midnight, later that day + * and the next midnight, a document with no value at all and one with only `d2`. + */ + private val nullifDocs = List( + """{"id":"n1","d":"2024-01-31","d2":"2024-02-01","ts":"2024-01-31T00:00:00Z"}""", + """{"id":"n2","d":"2024-02-01","d2":"2024-01-31","ts":"2024-01-31T10:00:00Z"}""", + """{"id":"n3","d":"2024-01-31","d2":"2024-01-30","ts":"2024-02-01T00:00:00Z"}""", + """{"id":"n4"}""", + """{"id":"n5","d2":"2024-01-31","ts":"2024-01-31T23:59:59.999Z"}""" + ) + + private def nullifAnswers( + n1: Expected, + n2: Expected, + n3: Expected, + n4: Expected, + n5: Expected + ): Map[String, Expected] = + Map("n1" -> n1, "n2" -> n2, "n3" -> n3, "n4" -> n4, "n5" -> n5) + + /** A DATE beside a date-TIME literal, a TIMESTAMP beside a date-only one: PostgreSQL reads the + * literal as the operand's type -- the day it names, the midnight it starts at -- and so does + * the comparison rule. The literal used to be parsed AGAIN on every document, by the operand's + * own format, which failed for these two shapes: the query failed and a computed column stored + * NULL. Every expected value is PostgreSQL 16's answer over the same documents. + */ + "NULLIF with a date or a timestamp literal of the other shape" should + "answer PostgreSQL's values in a query and STORE them in a computed column, in either argument order" in { + val columns = Seq( + ("c1", "DATE", "NULLIF(d, '2024-01-31 10:00:00')"), + ("c2", "TIMESTAMP", "NULLIF(ts, '2024-01-31')"), + ("c3", "DATE", "NULLIF('2024-01-31T10:00:00Z', d)"), + ("c4", "TIMESTAMP", "NULLIF('2024-01-31', ts)"), + ("c5", "DATE", "NULLIF(COALESCE(d, d2), '2024-01-31T10:00:00Z')") + ) + run( + s"CREATE TABLE $nullifTable (id KEYWORD, d DATE, d2 DATE, ts TIMESTAMP, " + + columns.map { case (c, t, e) => s"$c $t SCRIPT AS ($e)" }.mkString(", ") + + ", PRIMARY KEY (id))" + ) + bulk(nullifTable, nullifDocs) + val jan30 = At(start(on("2024-01-30"))) + val jan31 = At(start(on("2024-01-31"))) + val feb1 = At(start(on("2024-02-01"))) + val feb2 = At(start(on("2024-02-02"))) + val ts2 = At(at("2024-01-31T10:00:00Z")) + val ts3 = At(at("2024-02-01T00:00:00Z")) + val ts5 = At(at("2024-01-31T23:59:59.999Z")) + val day = At(start(today)) + val answers: Seq[(String, Map[String, Expected])] = Seq( + // the operand first + "NULLIF(d, '2024-01-31 10:00:00')" -> nullifAnswers(Null, feb1, Null, Null, Null), + "NULLIF(d, '2024-01-31T10:00:00Z')" -> nullifAnswers(Null, feb1, Null, Null, Null), + "NULLIF(ts, '2024-01-31')" -> nullifAnswers(Null, ts2, ts3, Null, ts5), + // the literal first: the answer is the literal, read as the operand's type + "NULLIF('2024-01-31T10:00:00Z', d)" -> nullifAnswers(Null, jan31, Null, jan31, jan31), + "NULLIF('2024-01-31 10:00:00', d)" -> nullifAnswers(Null, jan31, Null, jan31, jan31), + "NULLIF('2024-01-31', ts)" -> nullifAnswers(Null, jan31, jan31, jan31, jan31), + "NULLIF('2024-01-31T10:00:00Z', COALESCE(d, d2))" -> + nullifAnswers(Null, jan31, Null, jan31, Null), + // operands that are not columns + "NULLIF(COALESCE(d, d2), '2024-01-31T10:00:00Z')" -> + nullifAnswers(Null, feb1, Null, Null, Null), + "NULLIF(CAST(ts AS DATE), '2024-01-31T10:00:00Z')" -> + nullifAnswers(Null, Null, feb1, Null, Null), + "NULLIF(CAST(d AS TIMESTAMP), '2024-01-31')" -> nullifAnswers(Null, feb1, Null, Null, Null), + // PostgreSQL: `d + interval '1 day'` and `d + 1` give the same answers here + "NULLIF(DATE_ADD(d, INTERVAL 1 DAY), '2024-02-01')" -> + nullifAnswers(Null, feb2, Null, Null, Null), + "NULLIF(DATE_TRUNC(ts, DAY), '2024-01-31')" -> nullifAnswers(Null, Null, feb1, Null, Null), + "NULLIF(LEAST(d, d2), '2024-01-31T10:00:00Z')" -> + nullifAnswers(Null, Null, jan30, Null, Null), + "NULLIF(CURRENT_DATE, '2024-01-31T10:00:00Z')" -> nullifAnswers(day, day, day, day, day) + ) + // the clock: NOW() is never the literal's midnight, so the answer is the instant it was read + val clockSql = s"SELECT id, NULLIF(NOW(), '2024-01-31') AS x FROM $nullifTable" + val before = Instant.now().truncatedTo(ChronoUnit.MILLIS) + val clockRows = gatewayRows(clockSql) + val after = Instant.now() + val clock = clockSql -> clockRows.map { rs => + val got = byKey(rs, "id") + Seq("n1", "n2", "n3", "n4", "n5").flatMap { id => + differs(Within(before, after), got.getOrElse(id, null)).map(d => s"$id: $d") + } + } + verdict( + "NULLIF with a temporal literal", + answers.map { case (expression, values) => + comparisonVerdict(expression, values, nullifTable) + } ++ Seq( + clock, + storedVerdict( + nullifTable, + columns.map { case (column, _, expression) => + column -> answers.toMap.getOrElse(expression, fail(s"no answer for $expression")) + } + ) + ) + ) + } + + /** `CASE b WHEN '1'` compares the BOOLEAN with the boolean the literal spells, as PostgreSQL + * reads it (`'1'`, `'true'`, `'0'`, `'false'`), and `CASE b WHEN TRUE` with the constant. Each + * used to render the constant through the per-document conversion, `(true ? 1L : 0L)`, which + * Elasticsearch 6.8's Painless refuses to compile ("Extraneous conditional statement"): the + * query failed, and so did the CREATE TABLE of a computed column. A document with no `b` matches + * no `WHEN`. Every expected value is PostgreSQL 16's answer. + */ + "a simple CASE over a BOOLEAN, compared with a boolean constant" should + "answer PostgreSQL's values on every major, in a query and in a computed column" in { + run( + s"CREATE TABLE $booleanTable (id KEYWORD, b BOOLEAN, " + + "w1 INT SCRIPT AS (CASE b WHEN '1' THEN 1 ELSE 0 END), " + + "w2 INT SCRIPT AS (CASE b WHEN TRUE THEN 1 ELSE 0 END), PRIMARY KEY (id))" + ) + bulk( + booleanTable, + List("""{"id":"t1","b":true}""", """{"id":"t2","b":false}""", """{"id":"t3"}""") + ) + def answers(t1: Expected, t2: Expected, t3: Expected): Map[String, Expected] = + Map("t1" -> t1, "t2" -> t2, "t3" -> t3) + val whenTrue = answers(Num(1), Num(0), Num(0)) + val whenFalse = answers(Num(0), Num(1), Num(0)) + verdict( + "simple CASE over a BOOLEAN", + Seq( + comparisonVerdict("CASE b WHEN '1' THEN 1 ELSE 0 END", whenTrue, booleanTable), + comparisonVerdict("CASE b WHEN 'true' THEN 1 ELSE 0 END", whenTrue, booleanTable), + comparisonVerdict("CASE b WHEN 'false' THEN 1 ELSE 0 END", whenFalse, booleanTable), + comparisonVerdict("CASE b WHEN '0' THEN 1 ELSE 0 END", whenFalse, booleanTable), + comparisonVerdict("CASE b WHEN TRUE THEN 1 ELSE 0 END", whenTrue, booleanTable), + comparisonVerdict("CASE b WHEN FALSE THEN 1 ELSE 0 END", whenFalse, booleanTable), + comparisonVerdict( + "CASE b WHEN '1' THEN 1.5 ELSE 0 END", + answers(Num(1.5), Num(0), Num(0)), + booleanTable + ), + storedVerdict(booleanTable, Seq("w1" -> whenTrue, "w2" -> whenTrue)) + ) + ) + } + + /** A BOOLEAN column as a CASE condition is NULL on a document that has no value, and a NULL + * condition is not true: the branch does not apply, and the ELSE does -- or NULL, with no ELSE. + * It used to throw on that document (`param1 ? 1 : 0` over a null): the query failed, and a + * computed column stored NULL where SQL gives the ELSE value. Every expected value is PostgreSQL + * 16's answer. + */ + "a CASE WHEN over a BOOLEAN column" should + "take the ELSE branch on a document with no value, in a query, a WHERE and a computed column" in { + run( + s"CREATE TABLE $flagTable (id KEYWORD, flag BOOLEAN, n INT, " + + "c INT SCRIPT AS (CASE WHEN flag THEN 1 ELSE 0 END), " + + "c2 INT SCRIPT AS (CASE WHEN flag THEN n ELSE 0 END), PRIMARY KEY (id))" + ) + bulk( + flagTable, + List( + """{"id":"f1","flag":true,"n":5}""", + """{"id":"f2","flag":false,"n":6}""", + """{"id":"f3","n":7}""" + ) + ) + def answers(f1: Expected, f2: Expected, f3: Expected): Map[String, Expected] = + Map("f1" -> f1, "f2" -> f2, "f3" -> f3) + verdict( + "CASE WHEN over a BOOLEAN column", + Seq( + comparisonVerdict( + "CASE WHEN flag THEN 1 ELSE 0 END", + answers(Num(1), Num(0), Num(0)), + flagTable + ), + comparisonVerdict( + "CASE WHEN flag THEN n ELSE 0 END", + answers(Num(5), Num(0), Num(0)), + flagTable + ), + comparisonVerdict("CASE WHEN flag THEN 1 END", answers(Num(1), Null, Null), flagTable), + kept( + s"SELECT id FROM $flagTable WHERE CASE WHEN flag THEN 1 ELSE 0 END = 0", + Set("f2", "f3") + ), + storedVerdict( + flagTable, + Seq("c" -> answers(Num(1), Num(0), Num(0)), "c2" -> answers(Num(5), Num(0), Num(0))) + ) + ) + ) + } + + /** PostgreSQL 16's fixture for the operand tests below, over the same rows: an INT that is + * positive, negative and absent, a DOUBLE with a fraction, a KEYWORD holding digits, a BOOLEAN + * absent from one document, two DATEs and a TIMESTAMP. Created on first use. + */ + private lazy val operandTable: String = { + run( + s"CREATE TABLE $operandName (id KEYWORD, g KEYWORD, s KEYWORD, n INT, x DOUBLE, d DATE, " + + "d2 DATE, ts TIMESTAMP, b BOOLEAN, PRIMARY KEY (id))" + ) + bulk(operandName, operandDocs) + operandName + } + + private val operandDocs = List( + """{"id":"r1","g":"g1","s":"1","n":3,"x":0.25,"d":"2024-01-31","d2":"2024-02-01","ts":"2024-01-31T10:30:00Z","b":true}""", + """{"id":"r2","g":"g1","s":"5","n":5,"x":1.5,"d":"2024-03-01","d2":"2024-02-28","ts":"2024-03-01T06:00:00Z","b":false}""", + """{"id":"r3","g":"g2","s":"12","n":-2,"d":"2023-12-25","ts":"2023-12-25T23:59:59Z"}""", + """{"id":"r4","g":"g3","x":2.0,"d2":"2024-01-31","b":true}""", + """{"id":"r5","g":"g3","s":"3","n":10,"x":-1.25,"d":"2024-02-29","d2":"2024-02-29","ts":"2024-02-29T00:00:00Z","b":false}""" + ) + + private def answers( + r1: Expected, + r2: Expected, + r3: Expected, + r4: Expected, + r5: Expected + ): Map[String, Expected] = + Map("r1" -> r1, "r2" -> r2, "r3" -> r3, "r4" -> r4, "r5" -> r5) + + /** The statement is refused, by a message carrying every fragment, before anything runs. */ + private def refusedBy(sql: String, fragments: String*): (String, Either[String, Seq[String]]) = + sql -> (Try(Await.result(client.run(sql), 60.seconds)).toEither match { + case Right(ElasticFailure(error)) if fragments.forall(error.message.contains) => Right(Nil) + case Right(ElasticFailure(error)) => Right(Seq(s"refused as [${error.message}]")) + case Right(other) => Right(Seq(s"answered $other instead of the refusal")) + case Left(t) => Left(t.toString) + }) + + /** No table named `name` exists: DESCRIBE fails on it. */ + private def absent(name: String): (String, Either[String, Seq[String]]) = + s"DESCRIBE TABLE $name" -> (Try( + Await.result(client.run(s"DESCRIBE TABLE $name"), 60.seconds) + ).toEither match { + case Right(ElasticFailure(_)) => Right(Nil) + case other => Right(Seq(s"the table exists: $other")) + }) + + /** The type DESCRIBE reports for `field` of `table`, as a verdict. */ + private def declares(table: String, field: String, sqlType: String) = + s"DESCRIBE TABLE $table ($field)" -> described(table, field).map { t => + if (t.contains(sqlType)) Nil else Seq(s"$field is declared $t instead of $sqlType") + } + + /** `CREATE TABLE AS