From ebe867401aeddd94de9fd814aee64a03ebd348e7 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:36:10 +0200 Subject: [PATCH 1/4] feat(expr): contains, startsWith and endsWith escape LIKE wildcards The value is text, not a pattern: %, _ and the backslash are escaped. A string is escaped before it is bound; a param or expression is escaped in SQL with replace(). { insensitive: true } writes ILIKE. Postgres gets ESCAPE '\'; ClickHouse already escapes with a backslash. escapeLike() is the escape step on its own. Co-authored-by: Claude Opus 5.5 --- src/ch/expr.test-d.ts | 13 +++ src/ch/expr.ts | 65 +++++++++++++++ src/ch/param.ts | 3 + src/ch/text-match.test.ts | 62 +++++++++++++++ src/core.ts | 2 + src/expr.ts | 2 + tests/__snapshots__/core-sql.test.ts.snap | 96 +++++++++++++++++++++++ tests/core-cases.ts | 81 +++++++++++++++++++ 8 files changed, 324 insertions(+) create mode 100644 src/ch/text-match.test.ts diff --git a/src/ch/expr.test-d.ts b/src/ch/expr.test-d.ts index 77a7dde..0fd57c7 100644 --- a/src/ch/expr.test-d.ts +++ b/src/ch/expr.test-d.ts @@ -135,6 +135,19 @@ numExpr.ilike("%test%") // @ts-expect-error — .notLike() requires Expr numExpr.notLike("%test%") +expectTypeOf(strExpr.contains("50%")).toMatchTypeOf() +expectTypeOf(strExpr.startsWith("a", { insensitive: true })).toMatchTypeOf() +expectTypeOf(strExpr.endsWith(strExpr)).toMatchTypeOf() + +// @ts-expect-error — .contains() requires Expr +numExpr.contains("1") + +// @ts-expect-error — the value is a string, not a number +strExpr.startsWith(1) + +// @ts-expect-error — insensitive is a boolean +strExpr.endsWith("a", { insensitive: "yes" }) + // ColumnRef .get() — only valid for Map columns const mapRef = makeColumnRef<"Attrs", CHMap>("Attrs") diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 99773df..a268391 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -193,6 +193,14 @@ export interface Expr { like(this: Expr, pattern: string | Expr): Condition

notLike(this: Expr, pattern: string | Expr): Condition

ilike(this: Expr, pattern: string | Expr): Condition

+ /** + * Text search. The value is text, not a pattern: `%`, `_` and `\` in it + * match themselves. `{ insensitive: true }` ignores case (`ILIKE`). A value + * is a string or a param or expression of one. + */ + contains(this: Expr, value: string | Expr, options?: TextMatchOptions): Condition

+ startsWith(this: Expr, value: string | Expr, options?: TextMatchOptions): Condition

+ endsWith(this: Expr, value: string | Expr, options?: TextMatchOptions): Condition

// NULL and ranges /** `expr IS NULL`. */ @@ -227,6 +235,12 @@ export interface Expr { mod(this: Expr, n: R | Expr): Expr, P | Q> } +/** Options of `contains`, `startsWith` and `endsWith`. */ +export interface TextMatchOptions { + /** Ignore case: `ILIKE` instead of `LIKE`. Which characters fold is the database's choice; see docs/expressions.md. */ + readonly insensitive?: boolean | undefined +} + /** * What a comparison takes on its right: a value of the column's type (never * `null`), or an expression of it. @@ -387,6 +401,54 @@ const likePattern = (pattern: string | Expr): string => { ) } +/** The LIKE escape character of every text match. */ +const LIKE_ESCAPE = "\\" + +/** + * `value` with `%`, `_` and `\` escaped by a backslash, so a LIKE pattern + * built from it matches the text itself. ClickHouse and Postgres escape with a + * backslash by default; `contains` and friends also write `ESCAPE '\'` where + * the dialect is not ClickHouse. + */ +export const escapeLike = (value: string): string => value.replace(/[\\%_]/g, (char) => `${LIKE_ESCAPE}${char}`) + +type TextMatch = "contains" | "startsWith" | "endsWith" + +/** `value` escaped and wrapped in the wildcards of `match`, as SQL. */ +const textPattern = (value: string | Expr, match: TextMatch): string => { + const before = match === "contains" || match === "endsWith" ? "%" : "" + const after = match === "contains" || match === "startsWith" ? "%" : "" + if (typeof value === "string") return compile(boundString(`${before}${escapeLike(value)}${after}`)) + if (isExprLike(value)) { + // The value is only known at run time: escape it in SQL, the same three + // characters, escape character first. + const q = (text: string) => compile(str(text)) + let escaped = compile(value.toFragment()) + for (const char of [LIKE_ESCAPE, "%", "_"]) escaped = `replace(${escaped}, ${q(char)}, ${q(LIKE_ESCAPE + char)})` + return `(${[before && q(before), escaped, after && q(after)].filter(Boolean).join(" || ")})` + } + return fail( + new QueryBuilderError({ + code: "InvalidLiteral", + message: `a ${match} value must be a string or a string expression, not ${describeUntyped(value)}`, + }), + "NULL", + ) +} + +/** `expr LIKE pattern`, or `ILIKE` when insensitive, with the escape clause where the dialect needs one. */ +const textMatch = (fragment: SqlFragment, value: string | Expr, match: TextMatch, options?: TextMatchOptions): Condition => + makeCond( + known(() => { + const op = options?.insensitive === true ? "ILIKE" : "LIKE" + // Outside a compile, ClickHouse is the default dialect. Its backslash + // escape is built in; standard SQL has no default escape character. + const dialect = activeSqlSyntax()?.dialect ?? "clickhouse" + const escape = dialect === "clickhouse" ? "" : ` ESCAPE ${compile(str(LIKE_ESCAPE))}` + return `${compile(fragment)} ${op} ${textPattern(value, match)}${escape}` + }), + ) + const dateTimeLiteral = (value: DateTime.Utc): string => activeSqlSyntax()?.dateTimeLiteral(value) ?? compile(str(chDateTimeLiteral(value))) @@ -534,6 +596,9 @@ export function makeExpr( like: (pattern) => makeCond(known(() => `${compile(fragment)} LIKE ${likePattern(pattern)}`)), notLike: (pattern) => makeCond(known(() => `${compile(fragment)} NOT LIKE ${likePattern(pattern)}`)), ilike: (pattern) => makeCond(known(() => `${compile(fragment)} ILIKE ${likePattern(pattern)}`)), + contains: (value, options) => textMatch(fragment, value, "contains", options), + startsWith: (value, options) => textMatch(fragment, value, "startsWith", options), + endsWith: (value, options) => textMatch(fragment, value, "endsWith", options), in_: (...values) => inCond(fragment, "IN", values.map((v) => () => operand(v))), notIn: (...values) => inCond(fragment, "NOT IN", values.map((v) => () => operand(v))), diff --git a/src/ch/param.ts b/src/ch/param.ts index 0239cc8..f62e7f0 100644 --- a/src/ch/param.ts +++ b/src/ch/param.ts @@ -115,6 +115,9 @@ function makeParamMarker( like: raise, notLike: raise, ilike: raise, + contains: raise, + startsWith: raise, + endsWith: raise, isNull: raise, isNotNull: raise, between: raise, diff --git a/src/ch/text-match.test.ts b/src/ch/text-match.test.ts new file mode 100644 index 0000000..f753edf --- /dev/null +++ b/src/ch/text-match.test.ts @@ -0,0 +1,62 @@ +// `contains`, `startsWith` and `endsWith` match text, not a pattern: `%`, `_` +// and the escape character in the value are escaped, on every dialect. Live +// runs on ClickHouse and Postgres are the `text-match` core cases. + +import { describe, expect, it } from "@effect/vitest" +import { Cause, Effect, Exit } from "effect" +import * as CH from "../clickhouse" +import * as PG from "../postgres" + +const Events = CH.table("events", { + columns: { Id: CH.string, Name: CH.string, Note: CH.nullable(CH.string) }, + engine: CH.engine.mergeTree(), + orderBy: ["Id"], +}) +const Users = PG.table("users", { columns: { id: PG.text, name: PG.text }, primaryKey: ["id"] }) + +describe("text match", () => { + it("contains escapes LIKE wildcards in a ClickHouse value", () => { + const compiled = CH.compileUnsafe(CH.from(Events).select("Id").where(($) => [$.Name.contains("50%_off\\")])) + expect(compiled.sql).toContain(String.raw`Name LIKE '%50\\%\\_off\\\\%'`) + }) + + it("startsWith and endsWith put the wildcard on one side, also on a nullable column", () => { + const compiled = CH.compileUnsafe( + CH.from(Events).select("Id").where(($) => [$.Name.startsWith("a_"), $.Note.endsWith("%z")]), + ) + expect(compiled.sql).toContain(String.raw`Name LIKE 'a\\_%'`) + expect(compiled.sql).toContain(String.raw`Note LIKE '%\\%z'`) + }) + + it("insensitive is ILIKE", () => { + const compiled = CH.compileUnsafe(CH.from(Events).select("Id").where(($) => [$.Name.contains("Ab", { insensitive: true })])) + expect(compiled.sql).toContain(`Name ILIKE '%Ab%'`) + }) + + it("binds the escaped value on Postgres and writes the escape character", () => { + const compiled = PG.compileUnsafe( + PG.from(Users).select("id").where(($) => [$.name.contains("50%_", { insensitive: true }), $.name.startsWith("x")]), + ) + expect(compiled.sql).toContain(String.raw`"name" ILIKE $1 ESCAPE '\'`) + expect(compiled.sql).toContain(String.raw`"name" LIKE $2 ESCAPE '\'`) + expect(compiled.parameters).toEqual([String.raw`%50\%\_%`, "x%"]) + }) + + it("escapes a param in SQL, since its value is known only at run time", () => { + const query = PG.from(Users).select("id").where(($) => [$.name.endsWith(PG.param.string("q"))]) + const compiled = PG.compileUnsafe(query, { q: "a%" }) + expect(compiled.sql).toContain( + String.raw`"name" LIKE ('%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_')) ESCAPE '\'`, + ) + expect(compiled.parameters).toEqual(["a%"]) + }) + + it("escapeLike escapes the escape character first", () => { + expect(CH.escapeLike(String.raw`\%_x`)).toBe(String.raw`\\\%\_x`) + }) + + it.each([[1], [{}], [null]])("a value of %o fails as QueryBuilderError", (value) => { + const exit = Effect.runSyncExit(CH.compile(CH.from(Events).select("Id").where(($) => [$.Name.contains(value as never)]))) + expect(Exit.isFailure(exit) && String(Cause.squash(exit.cause))).toMatch(/contains value must be a string/) + }) +}) diff --git a/src/core.ts b/src/core.ts index b32adcf..749914a 100644 --- a/src/core.ts +++ b/src/core.ts @@ -25,6 +25,8 @@ export { type Condition, type Comparable, type MapValueOf, + type TextMatchOptions, + escapeLike, lit, rawExpr, untypedExpr, diff --git a/src/expr.ts b/src/expr.ts index 863419c..c3bb7e3 100644 --- a/src/expr.ts +++ b/src/expr.ts @@ -6,7 +6,9 @@ export { type Condition, type Expr, type MapValueOf, + type TextMatchOptions, and, + escapeLike, dynamicColumn, inExprList, inList, diff --git a/tests/__snapshots__/core-sql.test.ts.snap b/tests/__snapshots__/core-sql.test.ts.snap index fba9e28..7d3c9ba 100644 --- a/tests/__snapshots__/core-sql.test.ts.snap +++ b/tests/__snapshots__/core-sql.test.ts.snap @@ -494,6 +494,48 @@ SELECT } `; +exports[`core SQL (clickhouse) > text-match 1`] = ` +{ + "parameters": [], + "sql": "WITH texts AS ( +SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir')) +) +SELECT + texts.Id AS id + FROM texts + WHERE (texts.Body LIKE '%50\\\\%%' OR texts.Body LIKE 'a\\\\_%' OR texts.Body LIKE '%\\\\\\\\dir') + ORDER BY id ASC", +} +`; + +exports[`core SQL (clickhouse) > text-match-insensitive 1`] = ` +{ + "parameters": [], + "sql": "WITH texts AS ( +SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir')) +) +SELECT + texts.Id AS id + FROM texts + WHERE (texts.Body ILIKE '%AXB%' OR texts.Body ILIKE 'A\\\\_%' OR texts.Body ILIKE '%\\\\% OFF') + ORDER BY id ASC", +} +`; + +exports[`core SQL (clickhouse) > text-match-param 1`] = ` +{ + "parameters": [], + "sql": "WITH texts AS ( +SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir')) +) +SELECT + texts.Id AS id + FROM texts + WHERE (texts.Body LIKE ('%' || replace(replace(replace('0%', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_') || '%') OR texts.Body LIKE ('%' || replace(replace(replace('_b', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_')) OR texts.Body LIKE (replace(replace(replace('C:\\\\', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_') || '%')) + ORDER BY id ASC", +} +`; + exports[`core SQL (clickhouse) > union 1`] = ` { "parameters": [], @@ -1176,6 +1218,60 @@ SELECT } `; +exports[`core SQL (postgres) > text-match 1`] = ` +{ + "parameters": [ + "%50\\%%", + "a\\_%", + "%\\\\dir", + ], + "sql": "WITH "texts" AS ( +SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir')) AS v("c0", "c1") +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE ("texts"."Body" LIKE $1 ESCAPE '\\' OR "texts"."Body" LIKE $2 ESCAPE '\\' OR "texts"."Body" LIKE $3 ESCAPE '\\') + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > text-match-insensitive 1`] = ` +{ + "parameters": [ + "%AXB%", + "A\\_%", + "%\\% OFF", + ], + "sql": "WITH "texts" AS ( +SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir')) AS v("c0", "c1") +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE ("texts"."Body" ILIKE $1 ESCAPE '\\' OR "texts"."Body" ILIKE $2 ESCAPE '\\' OR "texts"."Body" ILIKE $3 ESCAPE '\\') + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (postgres) > text-match-param 1`] = ` +{ + "parameters": [ + "0%", + "_b", + "C:\\", + ], + "sql": "WITH "texts" AS ( +SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir')) AS v("c0", "c1") +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE ("texts"."Body" LIKE ('%' || replace(replace(replace($1, '\\', '\\\\'), '%', '\\%'), '_', '\\_') || '%') ESCAPE '\\' OR "texts"."Body" LIKE ('%' || replace(replace(replace($2, '\\', '\\\\'), '%', '\\%'), '_', '\\_')) ESCAPE '\\' OR "texts"."Body" LIKE (replace(replace(replace($3, '\\', '\\\\'), '%', '\\%'), '_', '\\_') || '%') ESCAPE '\\') + ORDER BY "id" ASC", +} +`; + exports[`core SQL (postgres) > union 1`] = ` { "parameters": [ diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 3698368..e55aadc 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -207,6 +207,34 @@ const p = (...names: string[]) => names.map((name) => `param:${name}`) const orgOrders = (ctx: CoreContext) => ctx.from(ctx.orders).where(($) => [$.OrgId.eq(CH.param.string("orgId"))]) +/** + * Text with LIKE wildcards and a backslash in it, for the text-match cases. Its + * own table and CTE, so the shared fixture and every other case stay as they are. + */ +const textRows: ReadonlyArray = [ + [1, "50% off"], + [2, "500 off"], + [3, "a_b"], + [4, "aXb"], + [5, "C:\\dir"], + [6, "C:dir"], +] +const texts = (ctx: CoreContext) => { + const table = + ctx.dialect === "clickhouse" + ? CH.table("texts", { external: true, columns: { Id: CH.uint32, Body: CH.string } }) + : PG.table("texts", { external: true, columns: { Id: PG.int4, Body: PG.text } }) + // A ClickHouse string literal escapes a backslash; a Postgres one does not. + const literal = (text: string) => + `'${(ctx.dialect === "clickhouse" ? text.replaceAll("\\", "\\\\") : text).replaceAll("'", "''")}'` + const tuples = textRows.map(([id, body]) => `(${id}, ${literal(body)})`).join(", ") + const body = + ctx.dialect === "clickhouse" + ? `SELECT * FROM values('Id UInt32, Body String', ${tuples})` + : `SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES ${tuples}) AS v("c0", "c1")` + return CH.from(table as CH.Table<"texts", { readonly Id: Col; readonly Body: Col }>).withCTE("texts", body) +} + export const coreCases: readonly CoreCase[] = [ { id: "select-columns", @@ -699,6 +727,59 @@ export const coreCases: readonly CoreCase[] = [ }, expected: [{ id: 1 }], }, + { + id: "text-match", + covers: e("contains", "startsWith", "endsWith"), + build: (ctx) => + ctx.compile( + texts(ctx) + .select(($) => ({ id: $.Id })) + .where(($) => [ + CH.or( + // Unescaped, `50%` would also match "500 off", `a_` "aXb", and `\dir` (`\d` escapes `d`) "C:dir". + $.Body.contains("50%"), + $.Body.startsWith("a_"), + $.Body.endsWith("\\dir"), + ), + ]) + .orderBy(["id", "asc"]), + ), + expected: [{ id: 1 }, { id: 3 }, { id: 5 }], + }, + { + id: "text-match-param", + covers: [], + build: (ctx) => + ctx.compile( + texts(ctx) + .select(($) => ({ id: $.Id })) + .where(($) => [ + CH.or($.Body.contains(CH.param.string("percent")), $.Body.endsWith(CH.param.string("under")), $.Body.startsWith(CH.param.string("slash"))), + ]) + .orderBy(["id", "asc"]), + // Escaped in SQL: the values reach the server as they are. + { percent: "0%", under: "_b", slash: "C:\\" }, + ), + expected: [{ id: 1 }, { id: 3 }, { id: 5 }], + }, + { + id: "text-match-insensitive", + covers: [], + build: (ctx) => + ctx.compile( + texts(ctx) + .select(($) => ({ id: $.Id })) + .where(($) => [ + CH.or( + $.Body.contains("AXB", { insensitive: true }), + $.Body.startsWith("A_", { insensitive: true }), + $.Body.endsWith("% OFF", { insensitive: true }), + ), + ]) + .orderBy(["id", "asc"]), + ), + expected: [{ id: 1 }, { id: 3 }, { id: 4 }], + }, ] /** Cases a dialect cannot run yet, each with the reason. Empty is the goal. */ From a3969a1f2a3bee1eecdd1f277712c104732f8808 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:36:10 +0200 Subject: [PATCH 2/4] docs: text search with contains, startsWith and endsWith Co-authored-by: Claude Opus 5.5 --- docs/expressions.md | 27 ++++++++++++++++++++++++++- docs/reference.md | 5 ++++- src/docs-examples.test.ts | 19 +++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/docs/expressions.md b/docs/expressions.md index c859b4c..3d2acb4 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -81,6 +81,31 @@ list is written as the constant it means, `1 = 0` for `in_()` and `1 = 1` for `n `.like(pattern)`, `.notLike(pattern)`, `.ilike(pattern)` are constrained by `this` to `Expr`, so calling them on a numeric column is a type error. +### Text search + +`.contains(value)`, `.startsWith(value)` and `.endsWith(value)` match text, not a pattern. `%`, +`_` and `\` in the value match themselves: the helper escapes them with a backslash and adds the +wildcards. Use them for a search box; use `.like()` only for a pattern you wrote. + +```ts +.where(($) => [$.OrgId.eq("org_123"), $.Name.contains("50%_off", { insensitive: true })]) +// ClickHouse: … AND events.Name ILIKE '%50\\%\\_off%' +// Postgres: … AND "events"."Name" ILIKE $1 ESCAPE '\' -- $1 = '%50\%\_off%' +``` + +- The value is a string, or a param or expression of one. A string is escaped before it is bound. + A param or expression is escaped in SQL (`replace(…)` three times), because its value is known + only when the query runs. +- Postgres gets an explicit `ESCAPE '\'`. ClickHouse uses the backslash as its escape character + already, so it gets none. +- `{ insensitive: true }` writes `ILIKE`. Which characters fold is the database's choice. + ClickHouse folds UTF-8 (`'ÉCOLE' ILIKE 'é%'` is true). Postgres folds by the database's + `LC_CTYPE`: a `C` locale (PGlite's default) folds ASCII only. +- An empty value matches every non-NULL row. A NULL column never matches. +- `escapeLike(text)` is the escape step on its own, for a pattern you build for `.like()`. + +_(Backed by `docs/expressions.md > Text search escapes LIKE wildcards`.)_ + ## Combining conditions ```ts @@ -139,7 +164,7 @@ unbounded read. The [optional-filter recipe](./recipes.md#optional-filters-and-e an explicit choice. Escaping SQL strings does not escape LIKE wildcards. `%` and `_` in a `.like()` argument are -patterns; use `.eq()` for exact text. Keep table names, aliases, and raw SQL under application +patterns; use `.eq()` for exact text and [`.contains()`](#text-search) to search for it. Keep table names, aliases, and raw SQL under application control: value escaping is not an allowlist for identifiers. ## Arithmetic diff --git a/docs/reference.md b/docs/reference.md index abc2d46..0fc18e6 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -113,6 +113,7 @@ time; see [Params and compilation](./params-and-compilation.md#what-each-kind-ac | `undecoded(column)` | the column as the driver sends it, typed as its wire form | | `inExprList(expr, exprs)` | Same for expression lists | | `notInList(expr, values)` | `expr NOT IN ('a', 'b')` | +| `escapeLike(text)` | `text` with `%`, `_` and `\` escaped, for a `.like()` pattern | | `not(condition)` | `NOT (…)` | | `and(...conds)` / `or(...conds)` | One flat `(… AND …)` / `(… OR …)`; skips `undefined`, returns `undefined` when none are left | | `dynamicColumn(name, t?)` | An `Expr` from a runtime column name — a `GROUP BY` alias | @@ -123,7 +124,9 @@ time; see [Params and compilation](./params-and-compilation.md#what-each-kind-ac `Expr` methods: `eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `in_`, `notIn`, `between`, `notBetween`, `isNull`, `isNotNull`, `like`, `notLike`, -`ilike` (string-only), and `add`, `sub`, `mul`, `div`, `mod` (number-only, **no parentheses**). `div` and `mod` decode +`ilike`, `contains`, `startsWith`, `endsWith` (string-only; the last three escape `%`, `_` and `\` +in the value and take `TextMatchOptions`, `{ insensitive?: boolean }`, see +[Text search](./expressions.md#text-search)), and `add`, `sub`, `mul`, `div`, `mod` (number-only, **no parentheses**). `div` and `mod` decode as `number | null` — ClickHouse sends `inf`/`nan` as JSON `null` — except by a numeric literal of magnitude ≥ 1 (`Quotient`), which keeps the dividend's nullability; use `ifNull(ifNotFinite(expr, 0), lit(0))` for a guaranteed number otherwise. diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index ef673be..c850fb2 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -18,6 +18,7 @@ import { Effect, Exit, Option, Schema } from "effect" import * as CH from "./ch/index" import * as T from "./ch/types" import * as CHE from "./clickhouse" +import * as PG from "./postgres" import { parseStatement, renderStatement, withSettings } from "./sql/statement" import { compileCHUnsafe, compileUnionUnsafe } from "./ch/compile" import { raw as rawFragment, compile as compileFragment } from "./sql/sql-fragment" @@ -278,6 +279,24 @@ describe("docs/expressions.md", () => { }), ) + it("Text search escapes LIKE wildcards", () => { + const search = (text: string) => + CH.from(Events) + .select(($) => ({ name: $.Name })) + .where(($) => [$.OrgId.eq("org_123"), $.Name.contains(text, { insensitive: true })]) + expect(oneLine(compileCHUnsafe(search("50%_off"), {}).sql)).toBe( + String.raw`SELECT events.Name AS name FROM events WHERE events.OrgId = 'org_123' AND events.Name ILIKE '%50\\%\\_off%'`, + ) + + const Users = PG.table("users", { columns: { id: PG.text, name: PG.text }, primaryKey: ["id"] }) + const compiled = PG.compileUnsafe(PG.from(Users).select("id").where(($) => [$.name.startsWith("50%")])) + expect(oneLine(compiled.sql)).toBe(String.raw`SELECT "users"."id" AS "id" FROM "users" WHERE "users"."name" LIKE $1 ESCAPE '\'`) + expect(compiled.parameters).toEqual([String.raw`50\%%`]) + + // `.like()` takes a pattern; escapeLike makes text safe inside one. + expect(CHE.escapeLike("50%_off")).toBe(String.raw`50\%\_off`) + }) + it("Optional predicates with when", () => { const build = (nameFilter?: string) => CH.from(Events) From da3869df16ecb6a1552d125782a5c882a57e6043 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:36:10 +0200 Subject: [PATCH 3/4] chore: changeset for text search Co-authored-by: Claude Opus 5.5 --- .changeset/text-search.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/text-search.md diff --git a/.changeset/text-search.md b/.changeset/text-search.md new file mode 100644 index 0000000..3c53de0 --- /dev/null +++ b/.changeset/text-search.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +String expressions add `.contains(value)`, `.startsWith(value)` and `.endsWith(value)`: text search that treats the value as text, not as a LIKE pattern. `%`, `_` and `\` in the value are escaped with a backslash. A string is escaped before it is bound; a param or expression is escaped in SQL with `replace()`. `{ insensitive: true }` writes `ILIKE`. Postgres gets an explicit `ESCAPE '\'`; ClickHouse already escapes with a backslash. `escapeLike(text)` is the escape step on its own, for a `.like()` pattern. Postgres and ClickHouse. From c1a79e13d71149187bac94c6c49254969f51f292 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 2 Nov 2026 16:39:32 +0100 Subject: [PATCH 4/4] feat(expr): text match keeps case semantics on SQLite and D1 SQLite's LIKE ignores ASCII case and it has no ILIKE, so contains/startsWith/ endsWith were case-insensitive there (and the insensitive form failed to compile). A dialect with likeIgnoresCase now gets GLOB (escaping *, ? and [) for a case-sensitive match and lower(x) LIKE lower(pattern) ESCAPE '\' for an insensitive one. The LIKE escape clause follows main's likeEscape dialect flag, so Postgres, whose LIKE escapes with a backslash already, no longer gets an explicit ESCAPE. The text-match core cases now run on SQLite and D1 too, with rows that a case-insensitive LIKE or an unescaped GLOB would also match. Co-Authored-By: Claude Opus 5.5 --- .changeset/text-search.md | 2 +- docs/expressions.md | 18 ++- docs/sqlite.md | 5 +- src/ch/dialect.ts | 1 + src/ch/expr.ts | 54 +++++--- src/ch/text-match.test.ts | 26 +++- src/docs-examples.test.ts | 11 +- src/sql/sql-syntax.ts | 6 + src/sqlite/dialect.ts | 1 + tests/__snapshots__/core-sql.test.ts.snap | 147 ++++++++++++++++++++-- tests/core-cases.ts | 42 +++++-- 11 files changed, 257 insertions(+), 56 deletions(-) diff --git a/.changeset/text-search.md b/.changeset/text-search.md index 3c53de0..29bcae6 100644 --- a/.changeset/text-search.md +++ b/.changeset/text-search.md @@ -2,4 +2,4 @@ "@maple-dev/effect-orm": minor --- -String expressions add `.contains(value)`, `.startsWith(value)` and `.endsWith(value)`: text search that treats the value as text, not as a LIKE pattern. `%`, `_` and `\` in the value are escaped with a backslash. A string is escaped before it is bound; a param or expression is escaped in SQL with `replace()`. `{ insensitive: true }` writes `ILIKE`. Postgres gets an explicit `ESCAPE '\'`; ClickHouse already escapes with a backslash. `escapeLike(text)` is the escape step on its own, for a `.like()` pattern. Postgres and ClickHouse. +String expressions add `.contains(value)`, `.startsWith(value)` and `.endsWith(value)`: text search that treats the value as text, not as a LIKE pattern. `%`, `_` and `\` in the value are escaped with a backslash. A string is escaped before it is bound; a param or expression is escaped in SQL with `replace()`. Matching is case-sensitive by default and `{ insensitive: true }` ignores case, on every dialect: ClickHouse and Postgres write `LIKE` / `ILIKE`; SQLite and D1, whose `LIKE` ignores ASCII case and which have no `ILIKE`, write `GLOB` (with `*`, `?` and `[` escaped) / `lower(x) LIKE lower(pattern) ESCAPE '\'`. `escapeLike(text)` is the escape step on its own, for a `.like()` pattern. diff --git a/docs/expressions.md b/docs/expressions.md index 3d2acb4..1087017 100644 --- a/docs/expressions.md +++ b/docs/expressions.md @@ -90,17 +90,23 @@ wildcards. Use them for a search box; use `.like()` only for a pattern you wrote ```ts .where(($) => [$.OrgId.eq("org_123"), $.Name.contains("50%_off", { insensitive: true })]) // ClickHouse: … AND events.Name ILIKE '%50\\%\\_off%' -// Postgres: … AND "events"."Name" ILIKE $1 ESCAPE '\' -- $1 = '%50\%\_off%' +// Postgres: … AND "events"."Name" ILIKE $1 -- $1 = '%50\%\_off%' +// SQLite, D1: … AND lower("events"."Name") LIKE lower(?) ESCAPE '\' -- ? = '%50\%\_off%' ``` - The value is a string, or a param or expression of one. A string is escaped before it is bound. A param or expression is escaped in SQL (`replace(…)` three times), because its value is known only when the query runs. -- Postgres gets an explicit `ESCAPE '\'`. ClickHouse uses the backslash as its escape character - already, so it gets none. -- `{ insensitive: true }` writes `ILIKE`. Which characters fold is the database's choice. - ClickHouse folds UTF-8 (`'ÉCOLE' ILIKE 'é%'` is true). Postgres folds by the database's - `LC_CTYPE`: a `C` locale (PGlite's default) folds ASCII only. +- The backslash is the escape character on every dialect: ClickHouse and Postgres use it by + default, and SQLite gets `ESCAPE '\'`, as `.like()` does there. +- Case-sensitive by default, also on SQLite and D1, whose `LIKE` ignores ASCII case. There the + helpers write `GLOB` instead (`"Name" GLOB ?` with `? = '*50%*'`), escaping GLOB's `*`, `?` and `[` the same + way. +- `{ insensitive: true }` writes `ILIKE`, or on SQLite and D1 (which have no `ILIKE`) a `LIKE` + of `lower()` on both sides. Which characters fold is the database's choice. ClickHouse folds + UTF-8 (`'ÉCOLE' ILIKE 'é%'` is true). Postgres folds by the database's `LC_CTYPE`: a `C` locale + (PGlite's default) folds ASCII only. SQLite's `lower()` folds ASCII only (without the ICU + extension). - An empty value matches every non-NULL row. A NULL column never matches. - `escapeLike(text)` is the escape step on its own, for a pattern you build for `.like()`. diff --git a/docs/sqlite.md b/docs/sqlite.md index 2cc50db..f3ee022 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -183,7 +183,10 @@ written as decimal text, because SQLite reads some decimals as a neighbouring do fraction compared with an `integer` column (`Amount > 9.75`) compiles as on Postgres. It refuses these unsupported forms at compile time: - `DISTINCT ON`, row locks and `ILIKE`. Use `LIKE`, which on SQLite ignores ASCII case - (`'acme' LIKE 'ACME'` is true, unlike Postgres and ClickHouse). + (`'acme' LIKE 'ACME'` is true, unlike Postgres and ClickHouse). For text search, + `.contains()`, `.startsWith()` and `.endsWith()` match case as on the other dialects (they + write `GLOB`), and `{ insensitive: true }` compares `lower()` of both sides; see + [Text search](./expressions.md#text-search). - Text containing U+0000. SQLite stores it, but its drivers end text there, so the value would read back cut short; the builder refuses it in literals and bound params. - `ON CONSTRAINT` conflict targets. Name the indexed columns instead. diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 10613ed..64d60b3 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -237,6 +237,7 @@ const checkedSyntax = (dialect: Dialect): SqlSyntax => ({ dialect: dialect.name, ilike: dialect.ilike, likeEscape: dialect.likeEscape, + likeIgnoresCase: dialect.likeIgnoresCase, inexactNumberText: dialect.inexactNumberText, integerComparesFraction: dialect.integerComparesFraction, quoteIdent: dialect.quoteIdent, diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 37acb67..d851b98 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -424,30 +424,40 @@ const likePattern = (pattern: string | Expr): string => { ) } +/** `ESCAPE '\'` where LIKE has no default escape, so `\%` means a literal `%` on every dialect. */ +const likeEscape = (): string => (activeSqlSyntax()?.likeEscape === true ? " ESCAPE '\\'" : "") + /** The LIKE escape character of every text match. */ const LIKE_ESCAPE = "\\" /** * `value` with `%`, `_` and `\` escaped by a backslash, so a LIKE pattern - * built from it matches the text itself. ClickHouse and Postgres escape with a - * backslash by default; `contains` and friends also write `ESCAPE '\'` where - * the dialect is not ClickHouse. + * built from it matches the text itself. Every dialect reads `\` as the LIKE + * escape: ClickHouse and Postgres by default, SQLite through the `ESCAPE '\'` + * that `like` writes there. */ export const escapeLike = (value: string): string => value.replace(/[\\%_]/g, (char) => `${LIKE_ESCAPE}${char}`) +/** `value` with GLOB's `[`, `*` and `?` each in brackets, so a GLOB pattern built from it matches the text itself. */ +const escapeGlob = (value: string): string => value.replace(/[[*?]/g, (char) => `[${char}]`) + type TextMatch = "contains" | "startsWith" | "endsWith" -/** `value` escaped and wrapped in the wildcards of `match`, as SQL. */ -const textPattern = (value: string | Expr, match: TextMatch): string => { - const before = match === "contains" || match === "endsWith" ? "%" : "" - const after = match === "contains" || match === "startsWith" ? "%" : "" - if (typeof value === "string") return compile(boundString(`${before}${escapeLike(value)}${after}`)) +/** `value` escaped and wrapped in the wildcards of `match`, as SQL: LIKE's `%`, or GLOB's `*`. */ +const textPattern = (value: string | Expr, match: TextMatch, glob: boolean): string => { + const any = glob ? "*" : "%" + const before = match === "contains" || match === "endsWith" ? any : "" + const after = match === "contains" || match === "startsWith" ? any : "" + if (typeof value === "string") return compile(boundString(`${before}${(glob ? escapeGlob : escapeLike)(value)}${after}`)) if (isExprLike(value)) { - // The value is only known at run time: escape it in SQL, the same three - // characters, escape character first. + // The value is only known at run time: escape it in SQL, the same + // characters, the escaping one (`\`, or GLOB's `[`) first. const q = (text: string) => compile(str(text)) + const replacements = glob + ? ["[", "*", "?"].map((char) => [char, `[${char}]`] as const) + : [LIKE_ESCAPE, "%", "_"].map((char) => [char, LIKE_ESCAPE + char] as const) let escaped = compile(value.toFragment()) - for (const char of [LIKE_ESCAPE, "%", "_"]) escaped = `replace(${escaped}, ${q(char)}, ${q(LIKE_ESCAPE + char)})` + for (const [char, replacement] of replacements) escaped = `replace(${escaped}, ${q(char)}, ${q(replacement)})` return `(${[before && q(before), escaped, after && q(after)].filter(Boolean).join(" || ")})` } return fail( @@ -459,20 +469,24 @@ const textPattern = (value: string | Expr, match: TextMatch): strin ) } -/** `expr LIKE pattern`, or `ILIKE` when insensitive, with the escape clause where the dialect needs one. */ +/** + * A text match that means the same on every dialect: `LIKE`, or `ILIKE` when + * insensitive. Where LIKE ignores case and there is no ILIKE (SQLite, whose + * LIKE folds ASCII unless `PRAGMA case_sensitive_like` is on), a sensitive + * match is a `GLOB` and an insensitive one a LIKE of `lower()` on both sides. + */ const textMatch = (fragment: SqlFragment, value: string | Expr, match: TextMatch, options?: TextMatchOptions): Condition => makeCond( known(() => { - const op = options?.insensitive === true ? "ILIKE" : "LIKE" - // Outside a compile, ClickHouse is the default dialect. Its backslash - // escape is built in; standard SQL has no default escape character. - const dialect = activeSqlSyntax()?.dialect ?? "clickhouse" - const escape = dialect === "clickhouse" ? "" : ` ESCAPE ${compile(str(LIKE_ESCAPE))}` - return `${compile(fragment)} ${op} ${textPattern(value, match)}${escape}` + const insensitive = options?.insensitive === true + if (activeSqlSyntax()?.likeIgnoresCase !== true) { + return `${compile(fragment)} ${insensitive ? "ILIKE" : "LIKE"} ${textPattern(value, match, false)}${likeEscape()}` + } + return insensitive + ? `lower(${compile(fragment)}) LIKE lower(${textPattern(value, match, false)})${likeEscape()}` + : `${compile(fragment)} GLOB ${textPattern(value, match, true)}` }), ) -/** `ESCAPE '\'` where LIKE has no default escape, so `\%` means a literal `%` on every dialect. */ -const likeEscape = (): string => (activeSqlSyntax()?.likeEscape === true ? " ESCAPE '\\'" : "") const dateTimeLiteral = (value: DateTime.Utc): string => activeSqlSyntax()?.dateTimeLiteral(value) ?? compile(str(chDateTimeLiteral(value))) diff --git a/src/ch/text-match.test.ts b/src/ch/text-match.test.ts index f753edf..d27362d 100644 --- a/src/ch/text-match.test.ts +++ b/src/ch/text-match.test.ts @@ -1,11 +1,12 @@ // `contains`, `startsWith` and `endsWith` match text, not a pattern: `%`, `_` // and the escape character in the value are escaped, on every dialect. Live -// runs on ClickHouse and Postgres are the `text-match` core cases. +// runs on every dialect are the `text-match` core cases. import { describe, expect, it } from "@effect/vitest" import { Cause, Effect, Exit } from "effect" import * as CH from "../clickhouse" import * as PG from "../postgres" +import * as SQLite from "../sqlite" const Events = CH.table("events", { columns: { Id: CH.string, Name: CH.string, Note: CH.nullable(CH.string) }, @@ -33,12 +34,12 @@ describe("text match", () => { expect(compiled.sql).toContain(`Name ILIKE '%Ab%'`) }) - it("binds the escaped value on Postgres and writes the escape character", () => { + it("binds the escaped value on Postgres, whose LIKE escapes with a backslash already", () => { const compiled = PG.compileUnsafe( PG.from(Users).select("id").where(($) => [$.name.contains("50%_", { insensitive: true }), $.name.startsWith("x")]), ) - expect(compiled.sql).toContain(String.raw`"name" ILIKE $1 ESCAPE '\'`) - expect(compiled.sql).toContain(String.raw`"name" LIKE $2 ESCAPE '\'`) + expect(compiled.sql).toContain(`"name" ILIKE $1\n`) + expect(compiled.sql).toMatch(/"name" LIKE \$2$/) expect(compiled.parameters).toEqual([String.raw`%50\%\_%`, "x%"]) }) @@ -46,11 +47,26 @@ describe("text match", () => { const query = PG.from(Users).select("id").where(($) => [$.name.endsWith(PG.param.string("q"))]) const compiled = PG.compileUnsafe(query, { q: "a%" }) expect(compiled.sql).toContain( - String.raw`"name" LIKE ('%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_')) ESCAPE '\'`, + String.raw`"name" LIKE ('%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_'))`, ) expect(compiled.parameters).toEqual(["a%"]) }) + it("on SQLite, whose LIKE ignores ASCII case, a sensitive match is a GLOB and an insensitive one lowers both sides", () => { + const Items = SQLite.table("items", { id: SQLite.integer, name: SQLite.text }) + const compiled = SQLite.compileUnsafe( + SQLite.from(Items).select("id").where(($) => [$.name.contains("a*[?_"), $.name.startsWith("50%", { insensitive: true })]), + ) + expect(compiled.sql).toContain(`"items"."name" GLOB ?\n`) + expect(compiled.sql).toContain(String.raw`lower("items"."name") LIKE lower(?) ESCAPE '\'`) + expect(compiled.parameters).toEqual(["*a[*][[][?]_*", String.raw`50\%%`]) + const param = SQLite.compileUnsafe( + SQLite.from(Items).select("id").where(($) => [$.name.endsWith(SQLite.param.string("q"))]), + { q: "x" }, + ) + expect(param.sql).toContain(`GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]'))`) + }) + it("escapeLike escapes the escape character first", () => { expect(CH.escapeLike(String.raw`\%_x`)).toBe(String.raw`\\\%\_x`) }) diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index c850fb2..c89fc2e 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -19,6 +19,7 @@ import * as CH from "./ch/index" import * as T from "./ch/types" import * as CHE from "./clickhouse" import * as PG from "./postgres" +import * as SQLite from "./sqlite" import { parseStatement, renderStatement, withSettings } from "./sql/statement" import { compileCHUnsafe, compileUnionUnsafe } from "./ch/compile" import { raw as rawFragment, compile as compileFragment } from "./sql/sql-fragment" @@ -290,9 +291,17 @@ describe("docs/expressions.md", () => { const Users = PG.table("users", { columns: { id: PG.text, name: PG.text }, primaryKey: ["id"] }) const compiled = PG.compileUnsafe(PG.from(Users).select("id").where(($) => [$.name.startsWith("50%")])) - expect(oneLine(compiled.sql)).toBe(String.raw`SELECT "users"."id" AS "id" FROM "users" WHERE "users"."name" LIKE $1 ESCAPE '\'`) + expect(oneLine(compiled.sql)).toBe(`SELECT "users"."id" AS "id" FROM "users" WHERE "users"."name" LIKE $1`) expect(compiled.parameters).toEqual([String.raw`50\%%`]) + // SQLite's LIKE ignores ASCII case: a case-sensitive match is a GLOB, an insensitive one lowers both sides. + const Items = SQLite.table("users", { id: SQLite.text, name: SQLite.text }) + const sqlite = SQLite.compileUnsafe(SQLite.from(Items).select("id").where(($) => [$.name.contains("50%"), $.name.startsWith("A_", { insensitive: true })])) + expect(oneLine(sqlite.sql)).toBe( + String.raw`SELECT "users"."id" AS "id" FROM "users" WHERE "users"."name" GLOB ? AND lower("users"."name") LIKE lower(?) ESCAPE '\'`, + ) + expect(sqlite.parameters).toEqual(["*50%*", String.raw`A\_%`]) + // `.like()` takes a pattern; escapeLike makes text safe inside one. expect(CHE.escapeLike("50%_off")).toBe(String.raw`50\%\_off`) }) diff --git a/src/sql/sql-syntax.ts b/src/sql/sql-syntax.ts index e87bb68..517a29a 100644 --- a/src/sql/sql-syntax.ts +++ b/src/sql/sql-syntax.ts @@ -19,6 +19,12 @@ export interface SqlSyntax { * to escape `%`, `_` and `\` on every dialect. */ readonly likeEscape?: boolean + /** + * True when LIKE ignores (ASCII) case by default (SQLite), so a + * case-sensitive text match cannot be a LIKE. `contains` and friends write + * GLOB there instead. + */ + readonly likeIgnoresCase?: boolean /** * True when the database parses some decimal literals to a neighbouring * double (SQLite), so a fraction must be bound rather than written as text. diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index f4a79f9..7a6c460 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -35,6 +35,7 @@ export const sqliteDialect: Dialect = { functions: "sqlite", ilike: false, likeEscape: true, + likeIgnoresCase: true, inexactNumberText: true, integerComparesFraction: true, quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`, diff --git a/tests/__snapshots__/core-sql.test.ts.snap b/tests/__snapshots__/core-sql.test.ts.snap index d2d4e39..0aab7b1 100644 --- a/tests/__snapshots__/core-sql.test.ts.snap +++ b/tests/__snapshots__/core-sql.test.ts.snap @@ -552,12 +552,12 @@ exports[`core SQL (clickhouse) > text-match 1`] = ` { "parameters": [], "sql": "WITH texts AS ( -SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir')) +SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) ) SELECT texts.Id AS id FROM texts - WHERE (texts.Body LIKE '%50\\\\%%' OR texts.Body LIKE 'a\\\\_%' OR texts.Body LIKE '%\\\\\\\\dir') + WHERE (texts.Body LIKE '%50\\\\%%' OR texts.Body LIKE 'a\\\\_%' OR texts.Body LIKE '%\\\\\\\\dir' OR texts.Body LIKE '%[x]%' OR texts.Body LIKE '%o?') ORDER BY id ASC", } `; @@ -566,12 +566,12 @@ exports[`core SQL (clickhouse) > text-match-insensitive 1`] = ` { "parameters": [], "sql": "WITH texts AS ( -SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir')) +SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) ) SELECT texts.Id AS id FROM texts - WHERE (texts.Body ILIKE '%AXB%' OR texts.Body ILIKE 'A\\\\_%' OR texts.Body ILIKE '%\\\\% OFF') + WHERE (texts.Body ILIKE '%AXB%' OR texts.Body ILIKE 'A\\\\_%' OR texts.Body ILIKE '%\\\\% OFF' OR texts.Body ILIKE '%[X]%') ORDER BY id ASC", } `; @@ -580,12 +580,12 @@ exports[`core SQL (clickhouse) > text-match-param 1`] = ` { "parameters": [], "sql": "WITH texts AS ( -SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir')) +SELECT * FROM values('Id UInt32, Body String', (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) ) SELECT texts.Id AS id FROM texts - WHERE (texts.Body LIKE ('%' || replace(replace(replace('0%', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_') || '%') OR texts.Body LIKE ('%' || replace(replace(replace('_b', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_')) OR texts.Body LIKE (replace(replace(replace('C:\\\\', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_') || '%')) + WHERE (texts.Body LIKE ('%' || replace(replace(replace('0%', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_') || '%') OR texts.Body LIKE ('%' || replace(replace(replace('_b', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_')) OR texts.Body LIKE (replace(replace(replace('C:\\\\', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_') || '%') OR texts.Body LIKE ('%' || replace(replace(replace('[x]', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_') || '%') OR texts.Body LIKE ('%' || replace(replace(replace('o?', '\\\\', '\\\\\\\\'), '%', '\\\\%'), '_', '\\\\_'))) ORDER BY id ASC", } `; @@ -1278,6 +1278,65 @@ SELECT } `; +exports[`core SQL (d1) > text-match 1`] = ` +{ + "parameters": [ + "*50%*", + "a_*", + "*\\dir", + "*[[]x]*", + "*o[?]", + ], + "sql": "WITH "texts" AS ( +SELECT column1 AS "Id", column2 AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE ("texts"."Body" GLOB ? OR "texts"."Body" GLOB ? OR "texts"."Body" GLOB ? OR "texts"."Body" GLOB ? OR "texts"."Body" GLOB ?) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (d1) > text-match-insensitive 1`] = ` +{ + "parameters": [ + "%AXB%", + "A\\_%", + "%\\% OFF", + "%[X]%", + ], + "sql": "WITH "texts" AS ( +SELECT column1 AS "Id", column2 AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE (lower("texts"."Body") LIKE lower(?) ESCAPE '\\' OR lower("texts"."Body") LIKE lower(?) ESCAPE '\\' OR lower("texts"."Body") LIKE lower(?) ESCAPE '\\' OR lower("texts"."Body") LIKE lower(?) ESCAPE '\\') + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (d1) > text-match-param 1`] = ` +{ + "parameters": [ + "0%", + "_b", + "C:\\", + "[x]", + "o?", + ], + "sql": "WITH "texts" AS ( +SELECT column1 AS "Id", column2 AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE ("texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]') || '*') OR "texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]')) OR "texts"."Body" GLOB (replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]') || '*') OR "texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]') || '*') OR "texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]'))) + ORDER BY "id" ASC", +} +`; + exports[`core SQL (d1) > union 1`] = ` { "parameters": [ @@ -2013,14 +2072,16 @@ exports[`core SQL (postgres) > text-match 1`] = ` "%50\\%%", "a\\_%", "%\\\\dir", + "%[x]%", + "%o?", ], "sql": "WITH "texts" AS ( -SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir')) AS v("c0", "c1") +SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) AS v("c0", "c1") ) SELECT "texts"."Id" AS "id" FROM "texts" - WHERE ("texts"."Body" LIKE $1 ESCAPE '\\' OR "texts"."Body" LIKE $2 ESCAPE '\\' OR "texts"."Body" LIKE $3 ESCAPE '\\') + WHERE ("texts"."Body" LIKE $1 OR "texts"."Body" LIKE $2 OR "texts"."Body" LIKE $3 OR "texts"."Body" LIKE $4 OR "texts"."Body" LIKE $5) ORDER BY "id" ASC", } `; @@ -2031,14 +2092,15 @@ exports[`core SQL (postgres) > text-match-insensitive 1`] = ` "%AXB%", "A\\_%", "%\\% OFF", + "%[X]%", ], "sql": "WITH "texts" AS ( -SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir')) AS v("c0", "c1") +SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) AS v("c0", "c1") ) SELECT "texts"."Id" AS "id" FROM "texts" - WHERE ("texts"."Body" ILIKE $1 ESCAPE '\\' OR "texts"."Body" ILIKE $2 ESCAPE '\\' OR "texts"."Body" ILIKE $3 ESCAPE '\\') + WHERE ("texts"."Body" ILIKE $1 OR "texts"."Body" ILIKE $2 OR "texts"."Body" ILIKE $3 OR "texts"."Body" ILIKE $4) ORDER BY "id" ASC", } `; @@ -2049,14 +2111,16 @@ exports[`core SQL (postgres) > text-match-param 1`] = ` "0%", "_b", "C:\\", + "[x]", + "o?", ], "sql": "WITH "texts" AS ( -SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir')) AS v("c0", "c1") +SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) AS v("c0", "c1") ) SELECT "texts"."Id" AS "id" FROM "texts" - WHERE ("texts"."Body" LIKE ('%' || replace(replace(replace($1, '\\', '\\\\'), '%', '\\%'), '_', '\\_') || '%') ESCAPE '\\' OR "texts"."Body" LIKE ('%' || replace(replace(replace($2, '\\', '\\\\'), '%', '\\%'), '_', '\\_')) ESCAPE '\\' OR "texts"."Body" LIKE (replace(replace(replace($3, '\\', '\\\\'), '%', '\\%'), '_', '\\_') || '%') ESCAPE '\\') + WHERE ("texts"."Body" LIKE ('%' || replace(replace(replace($1, '\\', '\\\\'), '%', '\\%'), '_', '\\_') || '%') OR "texts"."Body" LIKE ('%' || replace(replace(replace($2, '\\', '\\\\'), '%', '\\%'), '_', '\\_')) OR "texts"."Body" LIKE (replace(replace(replace($3, '\\', '\\\\'), '%', '\\%'), '_', '\\_') || '%') OR "texts"."Body" LIKE ('%' || replace(replace(replace($4, '\\', '\\\\'), '%', '\\%'), '_', '\\_') || '%') OR "texts"."Body" LIKE ('%' || replace(replace(replace($5, '\\', '\\\\'), '%', '\\%'), '_', '\\_'))) ORDER BY "id" ASC", } `; @@ -2731,6 +2795,65 @@ SELECT } `; +exports[`core SQL (sqlite) > text-match 1`] = ` +{ + "parameters": [ + "*50%*", + "a_*", + "*\\dir", + "*[[]x]*", + "*o[?]", + ], + "sql": "WITH "texts" AS ( +SELECT column1 AS "Id", column2 AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE ("texts"."Body" GLOB ? OR "texts"."Body" GLOB ? OR "texts"."Body" GLOB ? OR "texts"."Body" GLOB ? OR "texts"."Body" GLOB ?) + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > text-match-insensitive 1`] = ` +{ + "parameters": [ + "%AXB%", + "A\\_%", + "%\\% OFF", + "%[X]%", + ], + "sql": "WITH "texts" AS ( +SELECT column1 AS "Id", column2 AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE (lower("texts"."Body") LIKE lower(?) ESCAPE '\\' OR lower("texts"."Body") LIKE lower(?) ESCAPE '\\' OR lower("texts"."Body") LIKE lower(?) ESCAPE '\\' OR lower("texts"."Body") LIKE lower(?) ESCAPE '\\') + ORDER BY "id" ASC", +} +`; + +exports[`core SQL (sqlite) > text-match-param 1`] = ` +{ + "parameters": [ + "0%", + "_b", + "C:\\", + "[x]", + "o?", + ], + "sql": "WITH "texts" AS ( +SELECT column1 AS "Id", column2 AS "Body" FROM (VALUES (1, '50% off'), (2, '500 off'), (3, 'a_b'), (4, 'aXb'), (5, 'C:\\dir'), (6, 'C:dir'), (7, 'A_B'), (8, '[x]'), (9, 'x'), (10, 'who?'), (11, 'whom')) +) +SELECT + "texts"."Id" AS "id" + FROM "texts" + WHERE ("texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]') || '*') OR "texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]')) OR "texts"."Body" GLOB (replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]') || '*') OR "texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]') || '*') OR "texts"."Body" GLOB ('*' || replace(replace(replace(?, '[', '[[]'), '*', '[*]'), '?', '[?]'))) + ORDER BY "id" ASC", +} +`; + exports[`core SQL (sqlite) > union 1`] = ` { "parameters": [ diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 4655395..ec376d9 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -261,8 +261,10 @@ const orgOrders = (ctx: CoreContext) => ctx.from(ctx.orders).where(($) => [$.OrgId.eq(CH.param.string("orgId"))]) /** - * Text with LIKE wildcards and a backslash in it, for the text-match cases. Its - * own table and CTE, so the shared fixture and every other case stay as they are. + * Text with LIKE and GLOB wildcards and a backslash in it, for the text-match + * cases. Its own table and CTE, so the shared fixture and every other case stay + * as they are. Each matching row has a neighbour that an unescaped pattern, or + * one of the wrong case sensitivity, would also match. */ const textRows: ReadonlyArray = [ [1, "50% off"], @@ -271,20 +273,29 @@ const textRows: ReadonlyArray = [ [4, "aXb"], [5, "C:\\dir"], [6, "C:dir"], + [7, "A_B"], + [8, "[x]"], + [9, "x"], + [10, "who?"], + [11, "whom"], ] const texts = (ctx: CoreContext) => { const table = ctx.dialect === "clickhouse" ? CH.table("texts", { external: true, columns: { Id: CH.uint32, Body: CH.string } }) - : PG.table("texts", { external: true, columns: { Id: PG.int4, Body: PG.text } }) - // A ClickHouse string literal escapes a backslash; a Postgres one does not. + : ctx.dialect === "postgres" + ? PG.table("texts", { external: true, columns: { Id: PG.int4, Body: PG.text } }) + : SQLite.table("texts", { external: true, columns: { Id: SQLite.integer, Body: SQLite.text } }) + // A ClickHouse string literal escapes a backslash; a Postgres or SQLite one does not. const literal = (text: string) => `'${(ctx.dialect === "clickhouse" ? text.replaceAll("\\", "\\\\") : text).replaceAll("'", "''")}'` const tuples = textRows.map(([id, body]) => `(${id}, ${literal(body)})`).join(", ") const body = ctx.dialect === "clickhouse" ? `SELECT * FROM values('Id UInt32, Body String', ${tuples})` - : `SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES ${tuples}) AS v("c0", "c1")` + : ctx.dialect === "sqlite" || ctx.dialect === "d1" + ? `SELECT column1 AS "Id", column2 AS "Body" FROM (VALUES ${tuples})` + : `SELECT CAST("c0" AS int4) AS "Id", CAST("c1" AS text) AS "Body" FROM (VALUES ${tuples}) AS v("c0", "c1")` return CH.from(table as CH.Table<"texts", { readonly Id: Col; readonly Body: Col }>).withCTE("texts", body) } @@ -828,11 +839,15 @@ export const coreCases: readonly CoreCase[] = [ $.Body.contains("50%"), $.Body.startsWith("a_"), $.Body.endsWith("\\dir"), + // Case-insensitive (SQLite's LIKE), `a_` would also match "A_B". As GLOB + // patterns, `[x]` would match "x" and `o?` "whom". + $.Body.contains("[x]"), + $.Body.endsWith("o?"), ), ]) .orderBy(["id", "asc"]), ), - expected: [{ id: 1 }, { id: 3 }, { id: 5 }], + expected: [{ id: 1 }, { id: 3 }, { id: 5 }, { id: 8 }, { id: 10 }], }, { id: "text-match-param", @@ -842,13 +857,19 @@ export const coreCases: readonly CoreCase[] = [ texts(ctx) .select(($) => ({ id: $.Id })) .where(($) => [ - CH.or($.Body.contains(CH.param.string("percent")), $.Body.endsWith(CH.param.string("under")), $.Body.startsWith(CH.param.string("slash"))), + CH.or( + $.Body.contains(CH.param.string("percent")), + $.Body.endsWith(CH.param.string("under")), + $.Body.startsWith(CH.param.string("slash")), + $.Body.contains(CH.param.string("bracket")), + $.Body.endsWith(CH.param.string("question")), + ), ]) .orderBy(["id", "asc"]), // Escaped in SQL: the values reach the server as they are. - { percent: "0%", under: "_b", slash: "C:\\" }, + { percent: "0%", under: "_b", slash: "C:\\", bracket: "[x]", question: "o?" }, ), - expected: [{ id: 1 }, { id: 3 }, { id: 5 }], + expected: [{ id: 1 }, { id: 3 }, { id: 5 }, { id: 8 }, { id: 10 }], }, { id: "text-match-insensitive", @@ -862,11 +883,12 @@ export const coreCases: readonly CoreCase[] = [ $.Body.contains("AXB", { insensitive: true }), $.Body.startsWith("A_", { insensitive: true }), $.Body.endsWith("% OFF", { insensitive: true }), + $.Body.contains("[X]", { insensitive: true }), ), ]) .orderBy(["id", "asc"]), ), - expected: [{ id: 1 }, { id: 3 }, { id: 4 }], + expected: [{ id: 1 }, { id: 3 }, { id: 4 }, { id: 7 }, { id: 8 }], }, ]