From bb4fdb993d6151a731c1218f490df3f6a9ee0f50 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 16:56:16 +0200 Subject: [PATCH 1/8] feat(compile): keep compile state in the fiber context and read the dialect from CurrentDialect --- src/ch/compile.ts | 23 +++++------ src/ch/dialect.test.ts | 79 +++++++++++++++++++++++++++++++++++++- src/ch/dialect.ts | 36 ++++++++++------- src/ch/failure.ts | 30 ++++++++------- src/ch/subquery-context.ts | 22 +++++------ src/core.ts | 1 + src/postgres.ts | 21 +++++++--- src/sql/fiber-local.ts | 50 ++++++++++++++++++++++++ src/sql/literal-binder.ts | 20 +++++----- src/sql/render-tracker.ts | 35 ++++++++--------- src/sql/sql-syntax.ts | 23 +++++------ 11 files changed, 242 insertions(+), 98 deletions(-) create mode 100644 src/sql/fiber-local.ts diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..f3071d7 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -25,7 +25,8 @@ import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramColumnTypeO import { mergeResultSchemas } from "./define-fn" import { encodeValue } from "./literal" import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect" -import { Effect, Option, Schema } from "effect" +import { Context, Effect, Option, Schema } from "effect" +import { fiberLocal } from "../sql/fiber-local" import { QueryBuilderDefect, QueryBuilderError } from "./errors" import { withSubqueryCompiler } from "./subquery-context" import { tenantBoundOf, tenantColumnOf, tenantPredicatesOf, withTenantBound, type TenantPredicate } from "./tenant" @@ -1533,7 +1534,7 @@ function renderParams( const style = dialect.params const resolved = sql.replace(PARAM_PLACEHOLDER_PATTERN, (placeholder, kind: string, name: string) => { - const literal = boundLiterals?.get(name) + const literal = boundLiterals.get()?.get(name) if (!(name in params) && literal === undefined) { missing.push(name) return placeholder @@ -1629,7 +1630,11 @@ const VALUE_PARAM = "$$v" const LITERAL_PARAM = "$$l" /** The values bound by the enclosing compile, by param name; see `withBoundLiterals`. */ -let boundLiterals: Map | undefined +const boundLiterals = fiberLocal( + Context.Reference | undefined>("@maple-dev/effect-orm/internal/BoundLiterals", { + defaultValue: () => undefined, + }), +) /** * Run a compile with its literal values bound, on a dialect that binds params: @@ -1641,10 +1646,8 @@ function withBoundLiterals(dialect: Dialect, body: () => A): A { if (dialect.params._tag !== "bind" || activeLiteralBinder() !== undefined) return body() const values = new Map() const names = new Map() - const previous = boundLiterals - boundLiterals = values - try { - return withLiteralBinder((wire) => { + return boundLiterals.with(values, () => + withLiteralBinder((wire) => { const key = `${typeof wire}\0${JSON.stringify(wire)}` let name = names.get(key) if (name === undefined) { @@ -1653,10 +1656,8 @@ function withBoundLiterals(dialect: Dialect, body: () => A): A { values.set(name, { wire }) } return compileSqlFragment(param.of(insertWireValue, name).toFragment()) - }, body) - } finally { - boundLiterals = previous - } + }, body), + ) } const EMPTY_ROW = Schema.Struct({}) as unknown as CompiledQueryRowSchema diff --git a/src/ch/dialect.test.ts b/src/ch/dialect.test.ts index 2ac55b5..bb821cb 100644 --- a/src/ch/dialect.test.ts +++ b/src/ch/dialect.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from "@effect/vitest" -import { compile as compileFragment, str } from "../sql/sql-fragment" +import { Effect } from "effect" +import { compile as compileFragment, lazy, str } from "../sql/sql-fragment" import { compileCHUnsafe, compileUnionUnsafe } from "./compile" import type { Dialect } from "./dialect" import * as CH from "./index" +import * as PG from "../postgres" const numbered: Dialect = { ...CH.clickhouseDialect, @@ -223,3 +225,78 @@ describe("dialect identifiers and clauses", () => { expect(compileCHUnsafe(branch.format("JSON"), {}).sql).toMatch(/FORMAT JSON$/) }) }) + +describe("dialect from Effect context", () => { + const params = { orgId: "org_1", service: "api" } + + it.effect("compile reads CurrentDialect when no dialect is passed", () => + Effect.gen(function* () { + const compiled = yield* CH.compile(byService, params).pipe(Effect.provideService(CH.CurrentDialect, numbered)) + expect(compiled.sql).toContain("OrgId = $1") + expect(compiled.parameters).toEqual(["org_1", "api"]) + // An explicit dialect beats the one in context. + const explicit = yield* CH.compile(byService, params, { dialect: CH.clickhouseDialect }).pipe( + Effect.provideService(CH.CurrentDialect, numbered), + ) + expect(explicit.sql).toContain("OrgId = 'org_1'") + }), + ) + + it.effect("the /postgres compile defaults to Postgres, and also reads CurrentDialect", () => + Effect.gen(function* () { + const query = PG.from(PG.table("users", { columns: { email: PG.text } })).select("email").where(($) => [$.email.eq("a@x")]) + expect((yield* PG.compile(query)).sql).toContain(`"email" = $1`) + const inContext = yield* PG.compile(query).pipe(Effect.provideService(PG.CurrentDialect, standard)) + expect(inContext.sql).toContain("email = 'a@x'") + expect(PG.compileUnsafe(query).sql).toContain(`"email" = $1`) + }), + ) + + it.effect("compiles for two dialects in concurrent fibers", () => + Effect.gen(function* () { + const compileIn = (dialect: Dialect) => + Effect.gen(function* () { + yield* Effect.yieldNow + const compiled = yield* CH.compile(byService, params) + yield* Effect.yieldNow + return compiled + }).pipe(Effect.provideService(CH.CurrentDialect, dialect)) + const results = yield* Effect.all( + Array.from({ length: 20 }, (_, index) => compileIn(index % 2 === 0 ? numbered : standard)), + { concurrency: "unbounded" }, + ) + results.forEach((compiled, index) => { + if (index % 2 === 0) { + expect(compiled.sql).toContain("OrgId = $1") + expect(compiled.parameters).toEqual(["org_1", "api"]) + } else { + expect(compiled.sql).toContain("OrgId = 'org_1'") + expect(compiled.parameters).toEqual([]) + } + }) + }), + ) + + it("keeps the compile's dialect in its own fiber, not in module state", () => { + // While a numbered compile runs, a compile in another fiber sees none of it. + let inner: string | undefined + const outer = compileCHUnsafe( + CH.from(events) + .select(($) => ({ count: $.Count })) + .where(($) => [ + $.OrgId.eq(CH.param.string("orgId")), + CH.makeCond( + lazy(() => { + inner = Effect.runSync(Effect.sync(() => compileCHUnsafe(byService, params).sql)) + return "1 = 1" + }), + [], + ), + ]), + { orgId: "org_1" }, + { dialect: numbered }, + ) + expect(outer.sql).toContain("OrgId = $1") + expect(inner).toContain("OrgId = 'org_1'") + }) +}) diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 17b6c97..1a87cde 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -6,7 +6,8 @@ // written, how resolved param values reach the server, which clauses exist, and // how a portable param kind encodes. See `design/dialects.md`. -import type { Schema } from "effect" +import { Context, type Schema } from "effect" +import { fiberLocal } from "../sql/fiber-local" import { quoteClickHouseString } from "../sql/sql-fragment" import { activeSqlSyntax, withSqlSyntax, type SqlSyntax } from "../sql/sql-syntax" import { QueryBuilderError } from "./errors" @@ -175,28 +176,35 @@ export const clickhouseDialect: Dialect = { functions: "clickhouse", } -// The dialect of the enclosing compile, beside the syntax installed for the -// fragment renderer. Same save/restore discipline as `withSqlSyntax`. -let current: Dialect | undefined +/** + * The dialect a compile writes SQL for, when it is not passed as `options.dialect`. + * + * Provide it to an Effect (`Effect.provideService(CurrentDialect, postgresDialect)`) + * and every `compile` inside writes for that dialect. Each fiber has its own, so + * compiles for two dialects can run at the same time. A compile also installs its + * own dialect here for the queries nested in it. Absent, `compile` uses its + * entry's dialect: ClickHouse from `/clickhouse`, Postgres from `/postgres`. + */ +export const CurrentDialect = Context.Reference("@maple-dev/effect-orm/CurrentDialect", { + defaultValue: () => undefined, +}) + +// Beside the syntax installed for the fragment renderer, with the same +// save/restore discipline as `withSqlSyntax`. +const current = fiberLocal(CurrentDialect) /** The dialect of the enclosing compile, or ClickHouse outside one. */ -export const currentDialect = (): Dialect => current ?? clickhouseDialect +export const currentDialect = (): Dialect => current.get() ?? clickhouseDialect /** The dialect of the enclosing compile, or `undefined` outside one. */ -export const activeDialect = (): Dialect | undefined => current +export const activeDialect = (): Dialect | undefined => current.get() /** Run `body` with `dialect`'s syntax installed, literals checked as above. */ export function withDialect(dialect: Dialect, body: () => A): A { // A nested compile for the dialect already installed keeps the checked // syntax that is there instead of wrapping it again. - if (current === dialect && activeSqlSyntax() !== undefined) return body() - const previous = current - current = dialect - try { - return withSqlSyntax(checkedSyntax(dialect), body) - } finally { - current = previous - } + if (current.get() === dialect && activeSqlSyntax() !== undefined) return body() + return current.with(dialect, () => withSqlSyntax(checkedSyntax(dialect), body)) } /** `dialect.literal`, checked as above. For params written in after the diff --git a/src/ch/failure.ts b/src/ch/failure.ts index 56c45b5..4df25f3 100644 --- a/src/ch/failure.ts +++ b/src/ch/failure.ts @@ -2,30 +2,31 @@ // error in the slot the enclosing compile opened and returns placeholder SQL, and // the compile entry point turns that slot into the Effect's outcome. -import { Effect } from "effect" +import { Context, Effect } from "effect" +import { fiberLocal, runSyncHere } from "../sql/fiber-local" import { QueryBuilderDefect, type QueryBuilderError } from "./errors" export type CompileFailure = QueryBuilderError | QueryBuilderDefect -// Compilation is synchronous, so one slot per outermost compile is enough. -let slot: Array | undefined +// One slot per outermost compile, kept in the compiling fiber's context (see +// `fiber-local.ts`). +const slot = fiberLocal( + Context.Reference | undefined>("@maple-dev/effect-orm/internal/CompileFailures", { + defaultValue: () => undefined, + }), +) /** Record `failure` for the enclosing compile, and carry on with `placeholder`. */ export const fail = (failure: CompileFailure, placeholder: A): A => { - slot?.push(failure) + slot.get()?.push(failure) return placeholder } /** Run `body`, collecting what it records. A nested call leaves them to the outermost one. */ export function collectFailures(body: () => A): { readonly value: A; readonly failures: ReadonlyArray } { - if (slot !== undefined) return { value: body(), failures: [] } + if (slot.get() !== undefined) return { value: body(), failures: [] } const failures: Array = [] - slot = failures - try { - return { value: body(), failures } - } finally { - slot = undefined - } + return { value: slot.with(failures, body), failures } } /** @@ -43,5 +44,8 @@ export const compiled = (body: () => A): Effect.Effect return error === undefined ? Effect.succeed(value) : Effect.fail(error) }) -/** {@link compiled}, run at once: the unsafe edge, which throws the recorded failure. */ -export const compiledUnsafe = (body: () => A): A => Effect.runSync(compiled(body)) +/** + * {@link compiled}, run at once: the unsafe edge, which throws the recorded failure. + * It runs in the calling fiber's context, so a compile nested in another keeps its state. + */ +export const compiledUnsafe = (body: () => A): A => runSyncHere(compiled(body)) diff --git a/src/ch/subquery-context.ts b/src/ch/subquery-context.ts index a9a75b3..94505dd 100644 --- a/src/ch/subquery-context.ts +++ b/src/ch/subquery-context.ts @@ -1,20 +1,20 @@ +import { Context } from "effect" +import { fiberLocal } from "../sql/fiber-local" import type { CHQuery } from "./query" type RenderSubquery = (query: string | CHQuery) => string -// Compilation is synchronous. Save/restore makes nested compilations independent, -// including when a callback throws. No context survives across Effect executions. -let current: RenderSubquery | undefined +// Kept in the compiling fiber's context (see `fiber-local.ts`). Save/restore +// makes nested compilations independent, including when a callback throws. +const current = fiberLocal( + Context.Reference("@maple-dev/effect-orm/internal/SubqueryCompiler", { + defaultValue: () => undefined, + }), +) export function withSubqueryCompiler(render: RenderSubquery, body: () => A): A { - const previous = current - current = render - try { - return body() - } finally { - current = previous - } + return current.with(render, body) } export const renderSubquery = (query: string | CHQuery, fallback: RenderSubquery): string => - (current ?? fallback)(query) + (current.get() ?? fallback)(query) diff --git a/src/core.ts b/src/core.ts index b32adcf..bb557c5 100644 --- a/src/core.ts +++ b/src/core.ts @@ -149,6 +149,7 @@ export { } from "./ch/compile" export { + CurrentDialect, type Dialect, type DialectClauses, type DialectTransactions, diff --git a/src/postgres.ts b/src/postgres.ts index 1610096..ae0517f 100644 --- a/src/postgres.ts +++ b/src/postgres.ts @@ -11,7 +11,9 @@ import { compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH, } from "./ch/compile" +import { activeDialect, type Dialect } from "./ch/dialect" import { postgresDialect } from "./pg/dialect" +import { Effect } from "effect" export * from "./core" export * from "./pg/types" @@ -54,20 +56,27 @@ export { } from "./schema/pg-define" export { SchemaDefinitionError, type DefinitionProblem, type DdlExpr, type DdlKey } from "./schema/define" -/** `compile`, for Postgres unless `options.dialect` says otherwise. */ +/** The dialect a `/postgres` compile writes for: `options.dialect`, else `CurrentDialect`, else Postgres. */ +const pgOptions = (options: O) => ({ + ...options, + dialect: options?.dialect ?? activeDialect() ?? postgresDialect, +}) + +/** `compile`, for Postgres unless `options.dialect` or `CurrentDialect` says otherwise. */ // Typed through `any` and cast: `compileCH` is overloaded (queries and inserts), -// and an overloaded type gives an arrow's parameters no contextual type. +// and an overloaded type gives an arrow's parameters no contextual type. The +// dialect is read when the Effect runs, from the fiber that runs it. export const compile = ((query: any, params?: any, options?: any) => - compileCH(query, params, { ...options, dialect: options?.dialect ?? postgresDialect })) as typeof compileCH + Effect.suspend(() => compileCH(query, params, pgOptions(options)))) as typeof compileCH /** `compileUnsafe` for Postgres. */ export const compileUnsafe = ((query: any, params?: any, options?: any) => - compileCHUnsafe(query, params, { ...options, dialect: options?.dialect ?? postgresDialect })) as typeof compileCHUnsafe + compileCHUnsafe(query, params, pgOptions(options))) as typeof compileCHUnsafe /** `compileUnion` for Postgres. */ export const compileUnion: typeof compileUnionCH = (union, params, options) => - compileUnionCH(union, params, { ...options, dialect: options?.dialect ?? postgresDialect }) + Effect.suspend(() => compileUnionCH(union, params, pgOptions(options))) /** `compileUnionUnsafe` for Postgres. */ export const compileUnionUnsafe: typeof compileUnionUnsafeCH = (union, params, options) => - compileUnionUnsafeCH(union, params, { ...options, dialect: options?.dialect ?? postgresDialect }) + compileUnionUnsafeCH(union, params, pgOptions(options)) diff --git a/src/sql/fiber-local.ts b/src/sql/fiber-local.ts new file mode 100644 index 0000000..aa8dff7 --- /dev/null +++ b/src/sql/fiber-local.ts @@ -0,0 +1,50 @@ +// State of one compile, kept in the context of the fiber that runs it. +// +// Identifiers, literals and failures are written deep inside expression code +// that has no argument to carry the compile's state. A compile installs that +// state in its fiber's context for the length of a synchronous body instead, +// and the code below reads it back from the same fiber. Two compiles in two +// fibers never see each other's state, also when a fiber yields. There is no +// module-level state: outside any fiber, a value is installed by running the +// body in a fiber of its own. + +import { Context, Effect, Fiber } from "effect" + +/** A value installed for a synchronous body, in the context of the fiber that runs it. */ +export interface FiberLocal { + /** The value installed by the enclosing body in this fiber, or the reference's default. */ + readonly get: () => A + /** Run `body` with `value` installed. The previous value comes back after, also when `body` throws. */ + readonly with: (value: A, body: () => B) => B +} + +export const fiberLocal = (ref: Context.Reference): FiberLocal => { + const get = (): A => { + const fiber = Fiber.getCurrent() + return fiber === undefined ? ref.defaultValue() : fiber.getRef(ref) + } + const with_ = (value: A, body: () => B): B => { + const fiber = Fiber.getCurrent() + // A throw from `body` is the caller's: `runSync` gives it back as it was thrown. + if (fiber === undefined) return Effect.runSync(Effect.sync(() => with_(value, body))) + const previous = fiber.context + fiber.setContext(Context.add(previous, ref, value)) + try { + return body() + } finally { + fiber.setContext(previous) + } + } + return { get, with: with_ } +} + +/** + * Run `effect` at once, in the context of the fiber that calls this, if any. + * For the unsafe compile entry points: a compile nested in another compile's + * body keeps the enclosing compile's state, and one called from an Effect sees + * what that Effect provided. + */ +export const runSyncHere = (effect: Effect.Effect): A => { + const fiber = Fiber.getCurrent() + return fiber === undefined ? Effect.runSync(effect) : Effect.runSyncWith(fiber.context)(effect) +} diff --git a/src/sql/literal-binder.ts b/src/sql/literal-binder.ts index 9739632..69a7ca2 100644 --- a/src/sql/literal-binder.ts +++ b/src/sql/literal-binder.ts @@ -6,20 +6,22 @@ // shape is one statement whatever the values. `compile` installs a binder here // for the duration of the compile, as it installs the dialect's syntax. +import { Context } from "effect" +import { fiberLocal } from "./fiber-local" + /** Takes an encoded wire value; returns the SQL that stands for it. */ export type LiteralBinder = (wire: unknown) => string -let current: LiteralBinder | undefined +// Kept in the compiling fiber's context; see `fiber-local.ts`. +const current = fiberLocal( + Context.Reference("@maple-dev/effect-orm/internal/LiteralBinder", { + defaultValue: () => undefined, + }), +) export function withLiteralBinder(binder: LiteralBinder | undefined, body: () => A): A { - const previous = current - current = binder - try { - return body() - } finally { - current = previous - } + return current.with(binder, body) } /** The binder installed by the enclosing compile, if there is one. */ -export const activeLiteralBinder = (): LiteralBinder | undefined => current +export const activeLiteralBinder = (): LiteralBinder | undefined => current.get() diff --git a/src/sql/render-tracker.ts b/src/sql/render-tracker.ts index 314b244..495ab5c 100644 --- a/src/sql/render-tracker.ts +++ b/src/sql/render-tracker.ts @@ -12,8 +12,12 @@ // never mistaken for a scalar one wrapping a bare column. The checks built on // this can miss an error inside opaque SQL, never invent one. // -// Rendering is synchronous, and every entry point saves and restores, so a -// nested compile (a subquery) and a callback that throws leave no state behind. +// The track is kept in the rendering fiber's context (see `fiber-local.ts`), +// and every entry point saves and restores, so a nested compile (a subquery) +// and a callback that throws leave no state behind. + +import { Context } from "effect" +import { fiberLocal } from "./fiber-local" /** What one tracked render found. */ export interface RenderTrack { @@ -31,39 +35,30 @@ interface State { hiddenDepth: number } -let current: State | undefined +const current = fiberLocal( + Context.Reference("@maple-dev/effect-orm/internal/RenderTrack", { defaultValue: () => undefined }), +) /** Render `body` with a fresh track, returning both. */ export function track(body: () => A): [A, RenderTrack] { - const previous = current const state: State = { track: { aggregate: false, columns: new Set() }, aggregateDepth: 0, hiddenDepth: 0 } - current = state - try { - return [body(), state.track] - } finally { - current = previous - } + return [current.with(state, body), state.track] } /** Render `body` with no track: a nested query's columns are its own. */ export function untracked(body: () => A): A { - const previous = current - current = undefined - try { - return body() - } finally { - current = previous - } + return current.with(undefined, body) } /** A column identifier rendered. */ export function noteColumn(name: string): void { - if (current !== undefined && current.aggregateDepth === 0 && current.hiddenDepth === 0) current.track.columns.add(name) + const state = current.get() + if (state !== undefined && state.aggregateDepth === 0 && state.hiddenDepth === 0) state.track.columns.add(name) } /** Render an aggregate call: it marks the clause, and its arguments are not bare columns. */ export function inAggregate(render: () => string): string { - const state = current + const state = current.get() if (state === undefined) return render() if (state.hiddenDepth === 0) state.track.aggregate = true state.aggregateDepth++ @@ -76,7 +71,7 @@ export function inAggregate(render: () => string): string { /** Render a window (`… OVER (…)`) or SQL the builder did not write: nothing inside is counted. */ export function hidden(render: () => string): string { - const state = current + const state = current.get() if (state === undefined) return render() state.hiddenDepth++ try { diff --git a/src/sql/sql-syntax.ts b/src/sql/sql-syntax.ts index bf0fe19..42d51f8 100644 --- a/src/sql/sql-syntax.ts +++ b/src/sql/sql-syntax.ts @@ -2,11 +2,12 @@ // // Identifiers and literals are written while a query's callbacks run, deep // inside expression code that has no dialect argument to read. `compile` -// installs the dialect's syntax here for the duration of the compile instead, +// installs the dialect's syntax in its fiber for the duration of the compile instead, // the same way `withSubqueryCompiler` hands down the subquery renderer. A leaf // module so the fragment renderer can read it without importing the builder. -import type { DateTime } from "effect" +import { Context, type DateTime } from "effect" +import { fiberLocal } from "./fiber-local" export interface SqlSyntax { /** The dialect's name (`clickhouse`, `postgres`, …), for the few choices expression code makes per dialect. */ @@ -22,19 +23,15 @@ export interface SqlSyntax { readonly dateTimeLiteral: (value: DateTime.Utc) => string } -// Compilation is synchronous. Save/restore makes nested compilations -// independent, including when a callback throws. -let current: SqlSyntax | undefined +// Kept in the compiling fiber's context (see `fiber-local.ts`). Save/restore +// makes nested compilations independent, including when a callback throws. +const current = fiberLocal( + Context.Reference("@maple-dev/effect-orm/internal/SqlSyntax", { defaultValue: () => undefined }), +) export function withSqlSyntax(syntax: SqlSyntax, body: () => A): A { - const previous = current - current = syntax - try { - return body() - } finally { - current = previous - } + return current.with(syntax, body) } /** The syntax installed by the enclosing compile, if there is one. */ -export const activeSqlSyntax = (): SqlSyntax | undefined => current +export const activeSqlSyntax = (): SqlSyntax | undefined => current.get() From 046e3781ed9b69bf87699f9685235bc024aeceb0 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 16:57:45 +0200 Subject: [PATCH 2/8] docs: document CurrentDialect and the fiber-local compile state --- .changeset/dialect-in-context.md | 5 +++++ design/dialects.md | 5 ++++- docs/params-and-compilation.md | 22 +++++++++++++++++++++- docs/reference.md | 9 ++++++--- src/docs-examples.test.ts | 19 +++++++++++++++++++ 5 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 .changeset/dialect-in-context.md diff --git a/.changeset/dialect-in-context.md b/.changeset/dialect-in-context.md new file mode 100644 index 0000000..50515f8 --- /dev/null +++ b/.changeset/dialect-in-context.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`CurrentDialect`, a `Context.Reference`, sets the dialect a `compile` writes for when no `options.dialect` is given: `CH.compile(query, params).pipe(Effect.provideService(CH.CurrentDialect, postgresDialect))`. The order is `options.dialect`, then `CurrentDialect`, then the entry's dialect. A compile's state (its dialect, syntax, bound values and failures) is now kept in the context of the fiber that runs it, not in module state, so compiles for two dialects in concurrent fibers stay independent. diff --git a/design/dialects.md b/design/dialects.md index b02cba5..4a7ff35 100644 --- a/design/dialects.md +++ b/design/dialects.md @@ -46,7 +46,10 @@ What we deliberately do not copy: every `Str` fragment, column literal, and inline param. `compile` installs the dialect for the length of the compile (`withDialect`, the same save/restore pattern as `withSubqueryCompiler`), because literals are written inside the query callbacks, which - have no dialect argument. The `__PARAM_` safety is now enforced rather than assumed: a + have no dialect argument. The installed values live in the compiling fiber's context + (`src/sql/fiber-local.ts`), not in module state, so two compiles in two fibers cannot see + each other's dialect. The dialect is `CurrentDialect`, a `Context.Reference` a caller can + also provide. The `__PARAM_` safety is now enforced rather than assumed: a literal that contains the marker fails with `InvalidLiteral`, so a dialect with weaker escaping cannot turn a value into a placeholder. 3. **Identifier quoting (done).** Column refs are `Ident` fragments carrying their qualifier diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index 59c33af..f24fb9c 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -177,7 +177,7 @@ CH.compileUnsafe(query, params, options?) // CompiledQuery, throws | `options.rowSchema` | Effect `Schema` used by `decodeRows` / `decodeFirstRow` | | `options.skipFormat` | Omit a trailing `FORMAT` clause (used internally for subqueries) | | `options.deferParams` | Leave placeholders unresolved, for SQL spliced into an outer compile | -| `options.dialect` | Overrides the entry's dialect (`clickhouseDialect` from `/clickhouse`, `postgresDialect` from `/postgres`) | +| `options.dialect` | Overrides `CurrentDialect` and the entry's dialect (`clickhouseDialect` from `/clickhouse`, `postgresDialect` from `/postgres`) | Each dialect entry exports its own `compile`, defaulting to that dialect. Unions use `compileUnion(union, params)`. @@ -261,6 +261,26 @@ than `1`/`0`. | `clauses.parenthesizeUnionBranches` | Whether each `UNION ALL` branch is wrapped in parentheses | | `paramCodecs` | Per-kind codec overrides for `param.*` | +### The dialect in Effect context + +`CurrentDialect` is a `Context.Reference`. Provide it, and every `compile` in that Effect writes +for that dialect when no `options.dialect` is given. Each fiber has its own value, so compiles +for two dialects can run at the same time in one program: + +```ts +const compiled = yield* CH.compile(query, { orgId: "org_1" }).pipe( + Effect.provideService(CH.CurrentDialect, numbered), +) +// compiled.sql: ... WHERE OrgId = $1 +// compiled.parameters: ["org_1"] +``` + +The order is: `options.dialect`, then `CurrentDialect`, then the entry's dialect. The rest of a +compile's state (its syntax, bound values and failures) is kept in the fiber's context too, not in +module state. + +_(Backed by `docs/params-and-compilation.md > The dialect can come from Effect context`.)_ + Params are resolved by rewriting placeholders in the finished SQL, so a dialect's literals must never spell the param marker `__PARAM_`. ClickHouse writes it as `\x5F_PARAM_`; a literal that does contain the marker fails the compile with `InvalidLiteral` rather than being rewritten. diff --git a/docs/reference.md b/docs/reference.md index abc2d46..658914b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -87,7 +87,8 @@ to its own dialect; `options.dialect` overrides it. `Dialect`, `DialectClauses` and `ParamStyle` describe a database: how identifiers and literals are written, how params reach the server, and which clauses exist. Pass one as -`options.dialect`. `DialectTransactions` (with `IsolationLevel` and `TransactionSettings`) says +`options.dialect`, or provide it as `CurrentDialect`, a `Context.Reference` +that `compile` reads when no `options.dialect` is given. `DialectTransactions` (with `IsolationLevel` and `TransactionSettings`) says which transactions the database supports; see [Database](./database.md). See [Params and compilation](./params-and-compilation.md#dialects) and [Postgres](./postgres.md). @@ -405,7 +406,8 @@ Types: `WindowSpec`, `CompiledWindowSpec`, `WindowFrameBound`, `WindowRowsFrame` ### Dialect `compile` and the other compile functions default to `clickhouseDialect`: params written into the -SQL as ClickHouse literals. +SQL as ClickHouse literals. A `CurrentDialect` in context, or `options.dialect`, overrides +the default. --- @@ -496,7 +498,8 @@ unit union), `dateBin(seconds, ts)`, `now()`, `lower(x)`, `upper(x)`, `length(x) ### Dialect `compile` and the other compile functions default to `postgresDialect`: numbered `$n` -placeholders, double-quoted identifiers. +placeholders, double-quoted identifiers. A `CurrentDialect` in context, or `options.dialect`, overrides +the default. --- diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index ef673be..0db8255 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -588,6 +588,25 @@ describe("docs/params-and-compilation.md", () => { expect(oneLine(compileCHUnsafe(query, { orgId: "org_a" }).sql)).toContain("'org_a'") expect(oneLine(compileCHUnsafe(query, { orgId: "org_b" }).sql)).toContain("'org_b'") }) + + it.effect("The dialect can come from Effect context", () => + Effect.gen(function* () { + const numbered: CH.Dialect = { + ...CH.clickhouseDialect, + name: "numbered", + params: { _tag: "bind", placeholder: (index) => `$${index}`, reuse: true }, + } + const query = CH.from(Events) + .select(($) => ({ name: $.Name })) + .where(($) => [$.OrgId.eq(CH.param.string("orgId"))]) + + const compiled = yield* CH.compile(query, { orgId: "org_1" }).pipe( + Effect.provideService(CH.CurrentDialect, numbered), + ) + expect(oneLine(compiled.sql)).toContain("OrgId = $1") + expect(compiled.parameters).toEqual(["org_1"]) + }), + ) }) // decoding-results.md From 74b372a77523a1374ca28d557f5dca93b86652e1 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:09:57 +0200 Subject: [PATCH 3/8] feat(types): refuse a function of another dialect in the type A built-in function adds a DialectEntry for its set to its result's params. The entry travels with the params into the query, and compile, compileUnion and DatabaseApi.run check it against the target dialect. Functions of two sets in one query are refused everywhere. Also fix orderBy(callback): it kept only the first sort key's params. --- src/ch/any-boundaries.test-d.ts | 10 +- src/ch/compile.ts | 191 +++++++++++++++++----------- src/ch/deep-codecs.test.ts | 6 +- src/ch/dialect.test-d.ts | 123 ++++++++++++++++++ src/ch/dialect.ts | 11 +- src/ch/expr.test-d.ts | 11 +- src/ch/expr.ts | 53 +++++++- src/ch/functions/aggregate.ts | 42 +++--- src/ch/functions/array.ts | 22 ++-- src/ch/functions/builtin.ts | 17 ++- src/ch/functions/conditional.ts | 10 +- src/ch/functions/date-time.ts | 26 ++-- src/ch/functions/map.ts | 12 +- src/ch/functions/numeric.ts | 18 +-- src/ch/functions/string.ts | 26 ++-- src/ch/functions/window.ts | 6 +- src/ch/params-propagation.test-d.ts | 11 ++ src/ch/query.ts | 6 +- src/ch/soundness.test.ts | 4 + src/core.ts | 10 ++ src/database/database.ts | 29 +++-- src/pg/dialect.ts | 2 +- src/pg/functions.ts | 44 +++---- src/postgres.ts | 16 ++- 24 files changed, 483 insertions(+), 223 deletions(-) create mode 100644 src/ch/dialect.test-d.ts diff --git a/src/ch/any-boundaries.test-d.ts b/src/ch/any-boundaries.test-d.ts index 6c961ce..98ce29a 100644 --- a/src/ch/any-boundaries.test-d.ts +++ b/src/ch/any-boundaries.test-d.ts @@ -3,11 +3,11 @@ import { expectTypeOf } from "expect-type" import * as CH from "./index" const values = CH.arrayFilter("x -> x > 0", CH.arrayOf(CH.lit(1))) -expectTypeOf(values).toEqualTypeOf>>() +expectTypeOf(values).toEqualTypeOf, CH.ClickHouseOnly>>() const nullable = CH.arrayFilter("x -> isNotNull(x)", CH.arrayOf(CH.nullIf(CH.lit(1), 1))) // A handwritten predicate cannot prove that NULL has been removed. -expectTypeOf(nullable).toEqualTypeOf>>() -expectTypeOf(CH.arrayJoin(values)).toEqualTypeOf>() +expectTypeOf(nullable).toEqualTypeOf, CH.ClickHouseOnly>>() +expectTypeOf(CH.arrayJoin(values)).toEqualTypeOf>() // @ts-expect-error arrayFilter requires an array CH.arrayFilter("x -> x > 0", CH.lit(1)) // @ts-expect-error filtered numeric elements cannot become strings @@ -19,8 +19,8 @@ expectTypeOf>().toEqualTypeOf<{ }>() // Public overloads already protect toDateTime's implementation signature. -expectTypeOf(CH.toDateTime(CH.lit(1))).toEqualTypeOf>() -expectTypeOf(CH.toDateTime(CH.lit("2026-01-01"))).toEqualTypeOf>() +expectTypeOf(CH.toDateTime(CH.lit(1))).toEqualTypeOf>() +expectTypeOf(CH.toDateTime(CH.lit("2026-01-01"))).toEqualTypeOf>() // @ts-expect-error arrays cannot be converted to DateTime CH.toDateTime(values) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index f3071d7..ed3d3ec 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -15,7 +15,7 @@ import { isInsert, type AnyInsert, type CHInsert } from "./insert" import { isDelete, isUpdate, type CHDelete, type CHUpdate } from "./update" import type { Table } from "./table" import { createColumnAccessor, createQualifiedColumnAccessor, createJoinedColumnAccessor, sourceAlias } from "./query" -import { aliased, columnTypeOf, isExprLike, type Condition, type Expr, type ParamsSatisfied } from "./expr" +import { aliased, columnTypeOf, isExprLike, type Condition, type DialectSatisfied, type Expr, type ParamsSatisfied } from "./expr" import { raw, identPath, quoteIdent, quoteIdentPath, compile as compileSqlFragment, type SqlFragment } from "../sql/sql-fragment" import { splitTerminalClauses } from "../sql/terminal-clauses" import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder" @@ -584,32 +584,9 @@ const failedQuery = (): CompiledQuery< * `QueryBuilderDefect` its defect. {@link compileCHUnsafe} runs it with * `Effect.runSync`, for a fixture that should fail its test loudly. */ -export function compileCH< - Cols extends ColumnDefs, - Output extends Record, - Joins extends Record, - Route extends string | undefined, - Params = never, - const Given extends object = {}, - Decoded extends Output = Output, ->( - query: CHQuery & NeedsSelect & ParamsSatisfied, - /** Values for the query's `param.*` markers: required, and typed, when it has any. */ - params?: Given, - options?: { - skipFormat?: boolean - rowSchema?: CompiledQueryRowSchema - deferParams?: boolean - dialect?: Dialect - }, -): Effect.Effect, QueryBuilderError> -/** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ -export function compileCH( - insert: CHWrite & ParamsSatisfied, - params?: Given, - options?: InsertCompileOptions, -): Effect.Effect, QueryBuilderError> -export function compileCH( +export const compileCH: Compile<"clickhouse"> = compileQueryOrWrite as Compile<"clickhouse"> + +function compileQueryOrWrite( query: CHQuery | CHWrite, params?: Record, options?: any, @@ -617,59 +594,136 @@ export function compileCH( return compiled(() => compileCHRaw(query, params, options)) } +/** + * The type of {@link compileCH}, with `Default` the function set of the + * entry's own dialect: what the query's functions are checked against when no + * `options.dialect` is given. `/postgres` exports a `compile` of `Compile<"postgres">`. + * + * The query signature is the last one, so a refused query reports its own + * reason (a missing param, a function of another dialect) rather than why it + * is not a write. + */ +export interface Compile { + /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ + ( + insert: CHWrite & ParamsSatisfied & DialectSatisfied, + params?: Given, + options?: InsertCompileOptions, + ): Effect.Effect, QueryBuilderError> + < + Cols extends ColumnDefs, + Output extends Record, + Joins extends Record, + Route extends string | undefined, + Params = never, + const Given extends object = {}, + Decoded extends Output = Output, + Functions extends string = Default, + >( + query: CHQuery & + NeedsSelect & + ParamsSatisfied & + DialectSatisfied, + /** Values for the query's `param.*` markers: required, and typed, when it has any. */ + params?: Given, + options?: QueryCompileOptions, + ): Effect.Effect, QueryBuilderError> +} + +/** The type of {@link compileCHUnsafe}, with `Default` as in {@link Compile}. */ +export interface CompileUnsafe { + /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ + ( + insert: CHWrite & ParamsSatisfied & DialectSatisfied, + params?: Given, + options?: InsertCompileOptions, + ): CompiledQuery + < + Cols extends ColumnDefs, + Output extends Record, + Joins extends Record, + Route extends string | undefined, + Params = never, + const Given extends object = {}, + Decoded extends Output = Output, + Functions extends string = Default, + >( + query: CHQuery & + NeedsSelect & + ParamsSatisfied & + DialectSatisfied, + /** Values for the query's `param.*` markers: required, and typed, when it has any. */ + params?: Given, + options?: QueryCompileOptions, + ): CompiledQuery +} + /** A write statement: what `compile` takes besides a query. */ export type CHWrite = | AnyInsert | CHUpdate | CHDelete +/** + * What compiling a query takes. `dialect` is also what the query's functions + * are checked against in the type: its `Functions`, or the entry's own + * dialect when it is absent (see `DialectSatisfied`). + */ +export interface QueryCompileOptions { + readonly skipFormat?: boolean + readonly rowSchema?: CompiledQueryRowSchema + /** Leave `__PARAM_…__` placeholders in the SQL instead of resolving them. + * For fragments spliced into a larger query — a subquery condition — whose + * params are resolved by the outer compilation pass. */ + readonly deferParams?: boolean + /** The dialect to write for. The entry's dialect, or `CurrentDialect`, when omitted. */ + readonly dialect?: Dialect +} + /** What compiling a write takes: only the dialect. */ -export interface InsertCompileOptions { - readonly dialect?: Dialect +export interface InsertCompileOptions { + readonly dialect?: Dialect } -/** {@link compileCH} for a `UNION ALL`. */ -export const compileUnion = < +/** What compiling a union takes. */ +export interface UnionCompileOptions { + readonly rowSchema?: CompiledQueryRowSchema + readonly deferParams?: boolean + readonly dialect?: Dialect +} + +/** {@link compileCH} for a `UNION ALL`, with `Default` the function set of the entry's dialect. */ +export type CompileUnion = < Output extends Record, Params = never, const Given extends object = {}, + Functions extends string = Default, >( - union: CHUnionQuery & ParamsSatisfied, + union: CHUnionQuery & ParamsSatisfied & DialectSatisfied, params?: Given, - options?: { rowSchema?: CompiledQueryRowSchema; deferParams?: boolean; dialect?: Dialect }, -): Effect.Effect, QueryBuilderError> => - compiled(() => compileUnionRaw(union, params, options)) + options?: UnionCompileOptions, +) => Effect.Effect, QueryBuilderError> -export function compileCHUnsafe< - Cols extends ColumnDefs, +/** {@link compileUnion} without the Effect, with `Default` as there. */ +export type CompileUnionUnsafe = < Output extends Record, - Joins extends Record, - Route extends string | undefined, Params = never, const Given extends object = {}, - Decoded extends Output = Output, + Functions extends string = Default, >( - query: CHQuery & NeedsSelect & ParamsSatisfied, - /** Values for the query's `param.*` markers: required, and typed, when it has any. */ - params?: Given, - options?: { - skipFormat?: boolean - rowSchema?: CompiledQueryRowSchema - /** Leave `__PARAM_…__` placeholders in the SQL instead of resolving them. - * For fragments spliced into a larger query — a subquery condition — whose - * params are resolved by the outer compilation pass. */ - deferParams?: boolean - /** How params reach the server. ClickHouse literals when omitted. */ - dialect?: Dialect - }, -): CompiledQuery -/** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ -export function compileCHUnsafe( - insert: CHWrite & ParamsSatisfied, + union: CHUnionQuery & ParamsSatisfied & DialectSatisfied, params?: Given, - options?: InsertCompileOptions, -): CompiledQuery -export function compileCHUnsafe( + options?: UnionCompileOptions, +) => CompiledQuery + +/** {@link compileCH} for a `UNION ALL`. */ +export const compileUnion: CompileUnion<"clickhouse"> = (union, params, options) => + compiled(() => compileUnionRaw(union, params, options)) + +/** {@link compileCH} without the Effect: a failure throws. */ +export const compileCHUnsafe: CompileUnsafe<"clickhouse"> = compileQueryOrWriteUnsafe as CompileUnsafe<"clickhouse"> + +function compileQueryOrWriteUnsafe( query: CHQuery | CHWrite, params?: Record, options?: any, @@ -1365,21 +1419,8 @@ const unionExprsOf = ( // UNION ALL compilation -export function compileUnionUnsafe< - Output extends Record, - Params = never, - const Given extends object = {}, ->( - union: CHUnionQuery & ParamsSatisfied, - params?: Given, - options?: { - rowSchema?: CompiledQueryRowSchema - deferParams?: boolean - dialect?: Dialect - }, -): CompiledQuery { - return compiledUnsafe(() => compileUnionRaw(union, params, options)) -} +export const compileUnionUnsafe: CompileUnionUnsafe<"clickhouse"> = (union, params, options) => + compiledUnsafe(() => compileUnionRaw(union, params, options)) /** {@link compileCHRaw} for a union. */ export function compileUnionRaw>( diff --git a/src/ch/deep-codecs.test.ts b/src/ch/deep-codecs.test.ts index 31b7652..61abc86 100644 --- a/src/ch/deep-codecs.test.ts +++ b/src/ch/deep-codecs.test.ts @@ -10,7 +10,7 @@ const instant = DateTime.makeUnsafe("2026-01-02T00:00:00.789Z") describe("composed result codecs", () => { it("keeps nullable inserted array elements", () => { const value = CH.arrayPushFront(CH.arrayOf(CH.lit(1)), CH.nullIf(CH.lit(2), 2)) - expectTypeOf(value).toEqualTypeOf>>() + expectTypeOf(value).toEqualTypeOf, CH.ClickHouseOnly>>() expect(decode(value, [null, 1])).toEqual([null, 1]) const known = CH.arrayPushFront(CH.arrayOf(CH.lit(1)), CH.lit(2)) expect(Result.isFailure(Schema.decodeUnknownResult(known.schema!)([null]))).toBe(true) @@ -42,8 +42,8 @@ describe("composed result codecs", () => { it("propagates SQL NULL through string conversions", () => { const value = CH.nullIf(CH.lit(1), 1) - expectTypeOf(CH.toString(value)).toEqualTypeOf>() - expectTypeOf(CH.hex(value)).toEqualTypeOf>() + expectTypeOf(CH.toString(value)).toEqualTypeOf>() + expectTypeOf(CH.hex(value)).toEqualTypeOf>() expect(decode(CH.toString(value), null)).toBeNull() expect(decode(CH.hex(value), null)).toBeNull() expect(decode(CH.toString(CH.lit(1)), "1")).toBe("1") diff --git a/src/ch/dialect.test-d.ts b/src/ch/dialect.test-d.ts new file mode 100644 index 0000000..a19141e --- /dev/null +++ b/src/ch/dialect.test-d.ts @@ -0,0 +1,123 @@ +// Type-level tests: a built-in function of one dialect in a query compiled or +// run for another is a type error, not only a defect at compile time. + +import { Layer } from "effect" +import { expectTypeOf } from "expect-type" +import type * as SqlClient from "effect/sql/SqlClient" +import * as CH from "./index" +import * as PG from "../postgres" +import * as Db from "../database" +import type { Dialect } from "./dialect" +import type { DialectSatisfied, DialectEntry } from "./expr" + +const Events = CH.table("events", { Id: CH.string, Name: CH.string, Age: CH.uint64 }) +const Users = PG.table("users", { columns: { id: PG.text, name: PG.text, age: PG.int4 }, primaryKey: ["id"] }) + +// A ClickHouse function tags its expression, and the query that selects it. +expectTypeOf(CH.countIf(CH.lit(1).eq(1))).toEqualTypeOf>>() +expectTypeOf(PG.countIf(CH.lit(1).eq(1))).toEqualTypeOf>>() + +const chQuery = CH.from(Events).select(($) => ({ n: CH.countIf($.Age.gt(1)) })) +const pgQuery = PG.from(Users).select(($) => ({ n: PG.countIf($.age.gt(1)) })) +const portable = PG.from(Users).select(($) => ({ name: CH.nullIf($.name, "") })) + +// Each entry's default dialect accepts its own functions... +CH.compileUnsafe(chQuery) +PG.compileUnsafe(pgQuery) +void CH.compile(chQuery) +void PG.compile(pgQuery) +// ...and refuses the other's. +// @ts-expect-error -- countIf() is a ClickHouse function +PG.compileUnsafe(chQuery) +// @ts-expect-error -- countIf() is a ClickHouse function +void PG.compile(chQuery) +// @ts-expect-error -- countIf(*) FILTER is a Postgres function +CH.compileUnsafe(pgQuery) +// @ts-expect-error -- countIf(*) FILTER is a Postgres function +void CH.compile(pgQuery) + +// `options.dialect` decides, when given. +CH.compileUnsafe(pgQuery, {}, { dialect: PG.postgresDialect }) +PG.compileUnsafe(chQuery, {}, { dialect: CH.clickhouseDialect }) +// @ts-expect-error -- countIf() is a ClickHouse function +CH.compileUnsafe(chQuery, {}, { dialect: PG.postgresDialect }) + +// A dialect that does not name its function set is not checked by the type. +declare const custom: Dialect +CH.compileUnsafe(pgQuery, {}, { dialect: custom }) +PG.compileUnsafe(chQuery, {}, { dialect: custom }) + +// Portable functions and shared operators render for every dialect. +CH.compileUnsafe(portable) +PG.compileUnsafe(portable) + +// The message names both sides. +expectTypeOf, "postgres">>().toEqualTypeOf<{ + readonly wrongDialect: "this query uses ClickHouse functions; it cannot compile for the postgres dialect" +}>() +expectTypeOf, "clickhouse">>().toEqualTypeOf() +expectTypeOf>().toEqualTypeOf() +expectTypeOf, string>>().toEqualTypeOf() + +// Two dialects in one query: no dialect can compile it, a custom one included. +const mixed = PG.from(Users).select(($) => ({ a: PG.countIf($.age.gt(1)), b: CH.countIf($.age.gt(1)) })) +// @ts-expect-error -- ClickHouse and Postgres functions in one query +PG.compileUnsafe(mixed) +// @ts-expect-error -- ClickHouse and Postgres functions in one query +CH.compileUnsafe(mixed, {}, { dialect: custom }) +expectTypeOf | DialectEntry<"postgres">, string>>().toEqualTypeOf<{ + readonly mixedDialects: "this query uses both ClickHouse and Postgres functions; no dialect can compile it" +}>() + +// The tag travels with params, through every clause and into nested queries. +const inWhere = PG.from(Users).select("id").where(($) => [CH.hasToken($.name, "a")]) +// @ts-expect-error -- hasToken() is a ClickHouse function +PG.compileUnsafe(inWhere) +const inOrderBy = PG.from(Users).select("id").orderBy(($) => [[CH.lower($.name), "asc"], [CH.length($.name), "desc"]]) +// @ts-expect-error -- length() is a ClickHouse function +PG.compileUnsafe(inOrderBy) +const inHaving = PG.from(Users).select(($) => ({ n: PG.count() })).having(() => [CH.uniq(CH.lit(1)).gt(1)]) +// @ts-expect-error -- uniq() is a ClickHouse function +PG.compileUnsafe(inHaving) +const withParam = PG.from(Users) + .select("id") + .where(($) => [CH.hasToken($.name, CH.param.string("token"))]) +// @ts-expect-error -- hasToken() is a ClickHouse function, with a param +PG.compileUnsafe(withParam, { token: "a" }) +// @ts-expect-error -- the param is still required +CH.compileUnsafe(withParam, {}) +CH.compileUnsafe(withParam, { token: "a" }) +const outer = PG.from(Users).select("id").where(() => [CH.exists(chQuery)]) +// @ts-expect-error -- the subquery uses a ClickHouse function +PG.compileUnsafe(outer) +const joined = PG.from(Users).innerJoinQuery(chQuery, "c", ($, c) => c.n.eq($.age)).select("id") +// @ts-expect-error -- the joined subquery uses a ClickHouse function +PG.compileUnsafe(joined) +const union = CH.unionAll(PG.from(Users).select(($) => ({ n: PG.count() })), PG.from(Users).select(($) => ({ n: CH.count() }))) +// @ts-expect-error -- one branch uses a ClickHouse function +PG.compileUnionUnsafe(union) +// @ts-expect-error -- one branch uses a Postgres function +CH.compileUnionUnsafe(union, {}, { dialect: CH.clickhouseDialect }) + +// Writes too. +const update = PG.update(Users) + .set(($) => ({ name: CH.toString($.age) })) + .where(($) => [$.id.eq("u")]) +// @ts-expect-error -- toString() is a ClickHouse function +PG.compileUnsafe(update) + +// A `Database` built for a dialect takes only that dialect's queries. +declare const client: SqlClient.SqlClient +const pg = Db.fromSqlClient(client, { dialect: PG.postgresDialect }) +expectTypeOf(pg).toEqualTypeOf>() +void pg.run(pgQuery) +void pg.run(portable) +// @ts-expect-error -- countIf() is a ClickHouse function +void pg.run(chQuery) +// The `Database` in context has no dialect in its type; it still refuses a mixed query. +void Db.run(chQuery) +void Db.run(pgQuery) +// @ts-expect-error -- ClickHouse and Postgres functions in one query +void Db.run(mixed) +// A typed `Database` still provides the `Database` in context. +expectTypeOf(Layer.succeed(Db.Database, pg)).toEqualTypeOf>() diff --git a/src/ch/dialect.ts b/src/ch/dialect.ts index 1a87cde..0e58e65 100644 --- a/src/ch/dialect.ts +++ b/src/ch/dialect.ts @@ -131,7 +131,7 @@ export const noTransactions: DialectTransactions = { * finished text, so a value that spelled one would be rewritten too. A literal * that does is refused at compile time rather than trusted. */ -export interface Dialect extends Omit { +export interface Dialect extends Omit { /** Shown in errors. */ readonly name: string readonly params: ParamStyle @@ -150,12 +150,17 @@ export interface Dialect extends Omit { * `/clickhouse` entry's functions) or `postgres` (`@maple-dev/effect-orm/postgres`). * A built-in function from another set fails to compile. Absent means * unchecked: a custom dialect says which set it renders, if either. + * + * The type parameter is the same name at the type level: a query that uses + * a function of another set is a type error where it is compiled or run + * for a `Dialect<"postgres">` (see `DialectSatisfied`). A plain `Dialect` + * is not checked by the type. */ - readonly functions?: string + readonly functions?: Functions } /** ClickHouse, with params written into the SQL as literals. The default. */ -export const clickhouseDialect: Dialect = { +export const clickhouseDialect: Dialect<"clickhouse"> = { name: "clickhouse", quoteIdent: (name) => name, quoteString: quoteClickHouseString, diff --git a/src/ch/expr.test-d.ts b/src/ch/expr.test-d.ts index 77a7dde..1ac0895 100644 --- a/src/ch/expr.test-d.ts +++ b/src/ch/expr.test-d.ts @@ -4,6 +4,7 @@ import type { DateTime } from "effect" import { expectTypeOf } from "expect-type" import * as CH from "./index" import type { Expr, Condition } from "./expr" +import type { ClickHouseOnly } from "./functions/builtin" import type { ParamMarker } from "./param" import type { CHString, CHFloat64, CHMap } from "./types" import { makeColumnRef } from "./expr" @@ -30,11 +31,11 @@ expectTypeOf(CH.min(CH.lit(1))).toMatchTypeOf>() // when every contributing value is NULL, so the result must not narrow to the // non-null type. declare const nullableNum: Expr -expectTypeOf(CH.min(nullableNum)).toEqualTypeOf>() -expectTypeOf(CH.max(nullableNum)).toEqualTypeOf>() +expectTypeOf(CH.min(nullableNum)).toEqualTypeOf>() +expectTypeOf(CH.max(nullableNum)).toEqualTypeOf>() // ifNull with a non-nullable fallback strips the null the aggregate kept -expectTypeOf(CH.ifNull(nullableNum, CH.lit(0))).toEqualTypeOf>() +expectTypeOf(CH.ifNull(nullableNum, CH.lit(0))).toEqualTypeOf>() // any_ preserves generic expectTypeOf(CH.any(CH.lit("x"))).toMatchTypeOf>() @@ -211,8 +212,8 @@ expectTypeOf(numExpr.div(0)).toEqualTypeOf>() expectTypeOf(numExpr.mod(0)).toEqualTypeOf>() expectTypeOf(nullableNum.add(1).mul(2)).toEqualTypeOf>() expectTypeOf(CH.coalesce(nullableNum, CH.lit(0))).toEqualTypeOf>() -expectTypeOf(CH.ifNotFinite(nullableNum, 0)).toEqualTypeOf>() -expectTypeOf(CH.ifNull(CH.ifNotFinite(nullableNum, 0), CH.lit(0))).toEqualTypeOf>() +expectTypeOf(CH.ifNotFinite(nullableNum, 0)).toEqualTypeOf>() +expectTypeOf(CH.ifNull(CH.ifNotFinite(nullableNum, 0), CH.lit(0))).toEqualTypeOf>() // @ts-expect-error nullIf can return null const nonNullableString: Expr = CH.nullIf(CH.lit(""), "") // @ts-expect-error non-finite divisions serialize as null diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 99773df..caacb1f 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -101,22 +101,63 @@ export type ParamEntries = IsUnion extends tru type IsUnion = [T] extends [never] ? false : T extends unknown ? ([U] extends [T] ? false : true) : never /** `P` with every entry made optional. */ -export type OptionalParams

= P extends ParamEntry ? OptionalParamEntry : never +export type OptionalParams

= P extends ParamEntry ? OptionalParamEntry : P extends DialectEntry ? P : never /** The names some entry of `P` requires. */ type RequiredNames

= P extends ParamEntry ? (P extends { readonly optional: true } ? never : P["name"]) : never -/** The params of an expression, condition, or a union or array of them. */ +/** + * The function set an expression's SQL is written in, carried beside its + * params: a ClickHouse-only function such as `countIf` gives its result a + * `DialectEntry<"clickhouse">`, and every expression and query built on it + * carries the entry on. `compile` and `DatabaseApi.run` check the entries + * against the dialect they write for (see `DialectSatisfied`). A portable + * function or operator adds none. + */ +export interface DialectEntry { + readonly functions: Set +} + +/** The params of an expression, condition, or a union or array of them, and its dialect entries. */ export type ParamsIn = 0 extends 1 & X ? never : X extends { readonly _params?: (entries: infer P) => void } ? 0 extends 1 & P ? never - : [P] extends [ParamEntry] + : [P] extends [ParamEntry | DialectEntry] ? P : never : never +/** The function sets the entries `P` need: `never` when they need none. */ +export type DialectsIn

= P extends DialectEntry ? Set : never + +type FunctionSetLabel = Set extends "clickhouse" ? "ClickHouse" : Set extends "postgres" ? "Postgres" : Set + +/** + * `unknown` when the entries `P` can compile for a dialect whose function set + * is `Target`; otherwise a property saying why. A `Target` of `string` is a + * dialect that does not name its set, and is not checked. Functions of two sets + * in one query are refused for every target: no dialect renders both. + */ +export type DialectSatisfied = + IsUnion> extends true + ? { + readonly mixedDialects: `this query uses both ${JoinLabels>} functions; no dialect can compile it` + } + : string extends Target + ? unknown + : [DialectsIn

] extends [Target] + ? unknown + : { + readonly wrongDialect: `this query uses ${FunctionSetLabel>} functions; it cannot compile for the ${Target} dialect` + } + +/** `"ClickHouse and Postgres"` for a two-set union, in a fixed order. */ +type JoinLabels = [Sets] extends ["clickhouse" | "postgres"] + ? "ClickHouse and Postgres" + : FunctionSetLabel + /** The value a param needs: every entry's type for that name, intersected * per entry (a `boolean` entry stays `boolean`, not `true & false`). */ type ValueOf = @@ -132,7 +173,9 @@ type UnionToIntersection = (U extends unknown ? (u: U) => void : never) exten * The params object a set of entries asks for: one key per name. A name used * with two value types needs a value of both. */ -export type ParamsRecord

= [P] extends [never] +export type ParamsRecord

= ParamsRecordOf> + +type ParamsRecordOf

= [P] extends [never] ? {} : [P] extends [ParamEntry] ? Simplify< @@ -149,7 +192,7 @@ type Simplify = { [K in keyof T]: T[K] } & {} * otherwise a property spelling out the params object that is needed. Extra * keys are allowed, so one params object can serve several queries. */ -export type ParamsSatisfied = [P] extends [never] +export type ParamsSatisfied = [Extract] extends [never] ? unknown : Given extends ParamsRecord

? unknown diff --git a/src/ch/functions/aggregate.ts b/src/ch/functions/aggregate.ts index 1928192..b516622 100644 --- a/src/ch/functions/aggregate.ts +++ b/src/ch/functions/aggregate.ts @@ -8,7 +8,7 @@ import { type DateTime, Schema } from "effect" import * as T from "../types" import { schemaOf } from "../define-fn" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { compileTypedFnCall, defineFn, lazy } = builtins("clickhouse", "aggregate") @@ -24,7 +24,7 @@ export const count = defineFn<[], number>("count", T.uint64) export const avg = defineFn<[Expr], number | null>("avg", T.nullable(T.float64)) // A finite Float64 input can overflow during accumulation; the resulting // infinity decodes as NaN (see `overflowResultSchema`). -export const sum = (expr: Expr): Expr, Q> => +export const sum = (expr: Expr): Expr, Q | ClickHouseOnly> => compileTypedFnCall("sum", overflowResultSchema(expr), expr) // Condition-taking aggregates @@ -33,16 +33,16 @@ export const countIf = defineFn<[Condition], number>("countIf", T.uint64) export const sumIf = ( expr: Expr, condition: Condition, -): Expr, Q1 | Q2> => compileTypedFnCall("sumIf", overflowResultSchema(expr), expr, condition) +): Expr, Q1 | Q2 | ClickHouseOnly> => compileTypedFnCall("sumIf", overflowResultSchema(expr), expr, condition) export const avgIf = defineFn<[Expr, Condition], number | null>("avgIf", T.nullable(T.float64)) export const maxIf = ( expr: Expr, condition: Condition, -): Expr, Q1 | Q2> => compileTypedFnCall("maxIf", numericResultSchema(expr), expr, condition) +): Expr, Q1 | Q2 | ClickHouseOnly> => compileTypedFnCall("maxIf", numericResultSchema(expr), expr, condition) export const minIf = ( expr: Expr, condition: Condition, -): Expr, Q1 | Q2> => compileTypedFnCall("minIf", numericResultSchema(expr), expr, condition) +): Expr, Q1 | Q2 | ClickHouseOnly> => compileTypedFnCall("minIf", numericResultSchema(expr), expr, condition) // Generic aggregates (compileFnCall for type preservation) @@ -54,16 +54,16 @@ export const minIf = ( // here (as an earlier version did with `NonNullable`) lied to every caller // while `sameAs(0)` kept the nullable runtime codec. -export const min_ = (expr: Expr): Expr => compileTypedFnCall("min", schemaOf(expr), expr) +export const min_ = (expr: Expr): Expr => compileTypedFnCall("min", schemaOf(expr), expr) -export const max_ = (expr: Expr): Expr => compileTypedFnCall("max", schemaOf(expr), expr) +export const max_ = (expr: Expr): Expr => compileTypedFnCall("max", schemaOf(expr), expr) -export const any_ = (expr: Expr): Expr => compileTypedFnCall("any", schemaOf(expr), expr) +export const any_ = (expr: Expr): Expr => compileTypedFnCall("any", schemaOf(expr), expr) -export const anyIf = (expr: Expr, cond: Condition): Expr => +export const anyIf = (expr: Expr, cond: Condition): Expr => compileTypedFnCall("anyIf", schemaOf(expr), expr, cond) -export const uniq = (expr: Expr): Expr => compileTypedFnCall("uniq", T.uint64.schema, expr) +export const uniq = (expr: Expr): Expr => compileTypedFnCall("uniq", T.uint64.schema, expr) /** * `uniqIf(value, condition)` — distinct `value`s among the rows matching @@ -73,7 +73,7 @@ export const uniq = (expr: Expr): Expr => compile * `countIf` on a `ReplacingMergeTree`: un-merged duplicate rows for the same * key would inflate a `countIf` but not a `uniqIf` on that key. */ -export const uniqIf = (expr: Expr, cond: Condition): Expr => +export const uniqIf = (expr: Expr, cond: Condition): Expr => compileTypedFnCall("uniqIf", T.uint64.schema, expr, cond) /** @@ -83,10 +83,10 @@ export const uniqIf = (expr: Expr, cond: Condi * number sits next to the rows it counts: a facet count that disagrees with the * visible list reads as a bug, not as an approximation. */ -export const uniqExact = (expr: Expr): Expr => +export const uniqExact = (expr: Expr): Expr => compileTypedFnCall("uniqExact", T.uint64.schema, expr) -export const groupUniqArray = (expr: Expr): Expr, Q> => +export const groupUniqArray = (expr: Expr): Expr, Q | ClickHouseOnly> => compileTypedFnCall("groupUniqArray", arraySchemaOf(expr), expr) /** @@ -98,24 +98,24 @@ export const groupUniqArray = (expr: Expr): Expr(expr: Expr, Q>): Expr, Q> => +export const groupUniqArrayArray = (expr: Expr, Q>): Expr, Q | ClickHouseOnly> => compileTypedFnCall("groupUniqArrayArray", schemaOf>(expr), expr) /** `argMin(value, orderBy)` — the `value` from the row with the smallest `orderBy`. */ -export const argMin = (value: Expr, orderBy: Expr): Expr => +export const argMin = (value: Expr, orderBy: Expr): Expr => compileTypedFnCall("argMin", schemaOf(value), value, orderBy) /** `argMax(value, orderBy)` — the `value` from the row with the largest `orderBy`. */ -export const argMax = (value: Expr, orderBy: Expr): Expr => +export const argMax = (value: Expr, orderBy: Expr): Expr => compileTypedFnCall("argMax", schemaOf(value), value, orderBy) -export const argMaxMerge = (expr: Expr): Expr => +export const argMaxMerge = (expr: Expr): Expr => compileTypedFnCall("argMaxMerge", schemaOf(expr), expr) // Curried / parametric aggregates (handwritten — custom SQL syntax) export function quantile(q: number) { - return (expr: Expr): Expr => + return (expr: Expr): Expr => makeExpr(lazy(() => `quantile(${q})(${compile(expr.toFragment())})`), T.nullable(T.float64).schema) } @@ -127,7 +127,7 @@ export function quantile(q: number) { * curried shape: `groupUniqArrayIf(3)(x, cond)` → `groupUniqArrayIf(3)(x, cond)`. */ export function groupUniqArrayIf(maxSize: number) { - return (expr: Expr, cond: Condition): Expr, Q1 | Q2> => + return (expr: Expr, cond: Condition): Expr, Q1 | Q2 | ClickHouseOnly> => makeExpr( lazy(() => `groupUniqArrayIf(${Math.round(maxSize)})(` + @@ -164,7 +164,7 @@ export function windowFunnel(window: number, mode?: WindowFunnelMode) { return = ReadonlyArray>( timestamp: Expr, ...conditions: C - ): Expr> => { + ): Expr | ClickHouseOnly> => { // A failure, not a defect: the number of conditions is the number of steps a // funnel has, and that count comes from data as often as from source. const args = () => [timestamp.toFragment(), ...conditions.map((c) => c.toFragment())] @@ -194,7 +194,7 @@ export function sequenceMatch(pattern: string) { return = ReadonlyArray>( timestamp: Expr, ...conditions: C - ): Expr> => { + ): Expr | ClickHouseOnly> => { // An injection guard, so it is a failure: the pattern is embedded verbatim, // and "not user input" is a claim the caller is exactly who might get wrong. // Checked as the SQL renders, so it fails the compile that uses it. diff --git a/src/ch/functions/array.ts b/src/ch/functions/array.ts index 70d8fa5..5705b53 100644 --- a/src/ch/functions/array.ts +++ b/src/ch/functions/array.ts @@ -4,7 +4,7 @@ import type { Condition, Expr, ParamsIn } from "../expr" import { Schema } from "effect" import * as T from "../types" import { elementSchema, mergeResultSchemas, schemaOf } from "../define-fn" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { compileTypedFnCall, lazy } = builtins("clickhouse", "scalar") @@ -12,7 +12,7 @@ const { compileTypedFnCall, lazy } = builtins("clickhouse", "scalar") export function arrayOf> = ReadonlyArray>>( ...exprs: E & ReadonlyArray> -): Expr, ParamsIn> { +): Expr, ParamsIn | ClickHouseOnly> { const args = () => exprs.map((e) => compile(e.toFragment())).join(", ") const element = mergeResultSchemas(exprs.map((expr) => expr.schema)) return makeExpr(lazy(() => `[${args()}]`), element && Schema.Array(element)) @@ -23,7 +23,7 @@ export function arrayOf> = ReadonlyArray> | Expr>>( parts: Parts, sep: string, -): Expr> ? Parts[number] : Parts>> { +): Expr> ? Parts[number] : Parts> | ClickHouseOnly> { if (Array.isArray(parts)) { const arr = () => parts.map((p: Expr) => compile(p.toFragment())).join(", ") return makeExpr(lazy(() => `arrayStringConcat([${arr()}], ${compile(str(sep))})`), T.string.schema) @@ -35,31 +35,31 @@ export function arrayStringConcat ) } -export function arrayFilter(fn: string, arr: Expr, Q>): Expr, Q> { +export function arrayFilter(fn: string, arr: Expr, Q>): Expr, Q | ClickHouseOnly> { return makeExpr(lazy(() => `arrayFilter(${fn}, ${compile(arr.toFragment())})`), schemaOf>(arr)) } /** `arrayJoin` unnests, so the row value is one element of the array. */ -export const arrayJoin = (arr: Expr, Q>): Expr => +export const arrayJoin = (arr: Expr, Q>): Expr => compileTypedFnCall("arrayJoin", elementSchema(schemaOf>(arr)), arr) /** * Array functions that hand back the array they were given, reordered or * filtered — so the result decodes exactly as the input does. */ -export const arraySort = (arr: Expr, Q>): Expr, Q> => +export const arraySort = (arr: Expr, Q>): Expr, Q | ClickHouseOnly> => compileTypedFnCall("arraySort", schemaOf>(arr), arr) -export const arrayReverseSort = (arr: Expr, Q>): Expr, Q> => +export const arrayReverseSort = (arr: Expr, Q>): Expr, Q | ClickHouseOnly> => compileTypedFnCall("arrayReverseSort", schemaOf>(arr), arr) -export const arrayDistinct = (arr: Expr, Q>): Expr, Q> => +export const arrayDistinct = (arr: Expr, Q>): Expr, Q | ClickHouseOnly> => compileTypedFnCall("arrayDistinct", schemaOf>(arr), arr) export const arrayPushFront = ( arr: Expr, Q1>, element: Expr, -): Expr, Q1 | Q2> => { +): Expr, Q1 | Q2 | ClickHouseOnly> => { const item = mergeResultSchemas([elementSchema(arr.schema), element.schema]) return compileTypedFnCall("arrayPushFront", item && Schema.Array(item), arr, element) } @@ -69,9 +69,9 @@ export const arrayPushFront = ( export const arrayElement = ( arr: Expr, Q1>, index: number | Expr, -): Expr => compileTypedFnCall("arrayElement", elementSchema(schemaOf>(arr)), arr, index) +): Expr => compileTypedFnCall("arrayElement", elementSchema(schemaOf>(arr)), arr, index) -export function has(arr: Expr, Q1>, value: Expr | T): Condition { +export function has(arr: Expr, Q1>, value: Expr | T): Condition { const valueFragment = toFragment(value) return makeCond(lazy(() => `has(${compile(arr.toFragment())}, ${compile(valueFragment)})`)) } diff --git a/src/ch/functions/builtin.ts b/src/ch/functions/builtin.ts index 766d2a5..4637d3a 100644 --- a/src/ch/functions/builtin.ts +++ b/src/ch/functions/builtin.ts @@ -18,12 +18,19 @@ import { compile, known, type SqlFragment } from "../../sql/sql-fragment" import { hidden, inAggregate } from "../../sql/render-tracker" import { activeDialect } from "../dialect" import { QueryBuilderDefect } from "../errors" -import { type Condition, type Expr, makeCond, makeExpr, makeUntypedExpr, type ParamsIn, toFragment } from "../expr" +import { type Condition, type DialectEntry, type Expr, makeCond, makeExpr, makeUntypedExpr, type ParamsIn, toFragment } from "../expr" import type { FnResult } from "../define-fn" export type FunctionSet = "clickhouse" | "postgres" export type BuiltinKind = "scalar" | "aggregate" | "window" +/** The entry a built-in function of `Set` adds to its result: none for `portable`. */ +export type SetEntry = Set extends FunctionSet ? DialectEntry : never +/** What a ClickHouse function adds to its result's params: see `DialectEntry`. */ +export type ClickHouseOnly = DialectEntry<"clickhouse"> +/** What a Postgres function adds to its result's params: see `DialectEntry`. */ +export type PostgresOnly = DialectEntry<"postgres"> + const setLabel: Record = { clickhouse: "a ClickHouse function (from @maple-dev/effect-orm/clickhouse)", postgres: "a Postgres function (from @maple-dev/effect-orm/postgres)", @@ -41,7 +48,7 @@ const assertSet = (set: FunctionSet | "portable", name: string | undefined): voi } /** Built-in functions of one set and kind. `portable` renders for every dialect. */ -export function builtins(set: FunctionSet | "portable", kind: BuiltinKind) { +export function builtins(set: Set, kind: BuiltinKind) { const wrap = kind === "aggregate" ? inAggregate : kind === "window" ? hidden : (render: () => string): string => render() @@ -74,7 +81,7 @@ export function builtins(set: FunctionSet | "portable", kind: BuiltinKind) { function defineFn( name: string, result: FnResult, - ): (...args: A) => Expr> { + ): (...args: A) => Expr | SetEntry> { return (...args: A) => compileTypedFnCall( name, @@ -83,7 +90,9 @@ export function builtins(set: FunctionSet | "portable", kind: BuiltinKind) { ) } - function defineCondFn(name: string): (...args: A) => Condition> { + function defineCondFn( + name: string, + ): (...args: A) => Condition | SetEntry> { return (...args: A) => compileFnCallCond(name, ...args) } diff --git a/src/ch/functions/conditional.ts b/src/ch/functions/conditional.ts index a434ac1..8094952 100644 --- a/src/ch/functions/conditional.ts +++ b/src/ch/functions/conditional.ts @@ -3,7 +3,7 @@ import { compile } from "../../sql/sql-fragment" import type { Expr, Condition, ParamsIn } from "../expr" import { Schema } from "effect" import { firstTypedNonNull, mergeResultSchemas, numericResultSchema, schemaOf } from "../define-fn" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { compileTypedFnCall, lazy } = builtins("clickhouse", "scalar") const portable = builtins("portable", "scalar") @@ -15,13 +15,13 @@ export const if_ = ( cond: Condition, then_: Expr, else_: Expr, -): Expr => compileTypedFnCall("if", branchSchema(then_, else_), cond, then_, else_) +): Expr => compileTypedFnCall("if", branchSchema(then_, else_), cond, then_, else_) export function multiIf< T, const Cases extends ReadonlyArray]> = ReadonlyArray]>, Q = never, ->(cases: Cases & ReadonlyArray]>, else_: Expr): Expr | Q> { +>(cases: Cases & ReadonlyArray]>, else_: Expr): Expr | Q | ClickHouseOnly> { const parts = () => cases .map(([cond, val]) => `${compile(cond.toFragment())}, ${compile(val.toFragment())}`) .join(", ") @@ -58,7 +58,7 @@ export const coalesce = []>(...exprs: Args): Expr(expr: Expr, fallback: Expr): Expr => +export const ifNull = (expr: Expr, fallback: Expr): Expr => compileTypedFnCall("ifNull", firstTypedNonNull<[Expr, Expr], T>()(expr, fallback), expr, fallback) export function nullIf(expr: Expr, value: Expr | T): Expr { @@ -77,7 +77,7 @@ export function nullIf(expr: Expr, value: Expr export function ifNotFinite( expr: Expr, fallback: number | Expr, -): Expr, Q1 | Q2> { +): Expr, Q1 | Q2 | ClickHouseOnly> { return makeExpr>( lazy(() => `ifNotFinite(${compile(expr.toFragment())}, ${compile(toFragment(fallback))})`), numericResultSchema(expr), diff --git a/src/ch/functions/date-time.ts b/src/ch/functions/date-time.ts index a1d00b7..6d6cb17 100644 --- a/src/ch/functions/date-time.ts +++ b/src/ch/functions/date-time.ts @@ -4,7 +4,7 @@ import { schemaOf } from "../define-fn" import { str, compile } from "../../sql/sql-fragment" import type { Expr } from "../expr" import * as T from "../types" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { lazy } = builtins("clickhouse", "scalar") @@ -27,7 +27,7 @@ const sameDateTime = (col: Expr) => export function toStartOfInterval( col: DateTimeExpr, seconds: number | Expr, -): DateTimeExpr { +): DateTimeExpr { const secStr = () => typeof seconds === "number" ? String(Math.round(seconds)) @@ -47,7 +47,7 @@ export function toStartOfInterval( col: DateTimeExpr, -): DateTimeExpr { +): DateTimeExpr { return makeExpr(lazy(() => `toStartOfHour(${compile(col.toFragment())})`), sameDateTime(col)) } @@ -58,7 +58,7 @@ export function toStartOfHour */ export function toStartOfMinute( col: DateTimeExpr, -): DateTimeExpr { +): DateTimeExpr { return makeExpr(lazy(() => `toStartOfMinute(${compile(col.toFragment())})`), sameDateTime(col)) } @@ -67,7 +67,7 @@ export function toStartOfMinute(col: DateTimeExpr): Expr { +export function toHour(col: DateTimeExpr): Expr { return makeExpr(lazy(() => `toHour(${compile(col.toFragment())})`), T.uint8.schema) } @@ -77,7 +77,7 @@ export function toHour(col: DateTimeExpr): Expr(col: DateTimeExpr): Expr { +export function toUnixTimestamp(col: DateTimeExpr): Expr { return makeExpr(lazy(() => `toUnixTimestamp(${compile(col.toFragment())})`), T.uint32.schema) } @@ -85,14 +85,14 @@ export function toUnixTimestamp(col: DateTimeExpr): * `toUnixTimestamp64Nano(expr)` — convert DateTime64 to a nanosecond epoch. * Used for counter-rate delta windows where sub-second scrape spacing matters. */ -export function toUnixTimestamp64Nano(col: DateTimeExpr): Expr { +export function toUnixTimestamp64Nano(col: DateTimeExpr): Expr { return makeExpr(lazy(() => `toUnixTimestamp64Nano(${compile(col.toFragment())})`), T.uint64.schema) } export function intervalSub( col: DateTimeExpr, seconds: number | Expr, -): DateTimeExpr { +): DateTimeExpr { const secStr = () => typeof seconds === "number" ? String(Math.round(seconds)) @@ -104,7 +104,7 @@ export function intervalSub( col: DateTimeExpr, seconds: number | Expr, -): DateTimeExpr { +): DateTimeExpr { const secStr = () => typeof seconds === "number" ? String(Math.round(seconds)) @@ -113,7 +113,7 @@ export function intervalAdd(col: DateTimeExpr, format: string): Expr { +export function formatDateTime(col: DateTimeExpr, format: string): Expr { return makeExpr( lazy(() => `formatDateTime(${compile(col.toFragment())}, ${compile(str(format))})`), T.string.schema, @@ -126,9 +126,9 @@ export function formatDateTime(col: DateTimeExpr, f * require a Date/DateTime/DateTime64 argument and won't implicitly parse a * string literal. */ -export function toDateTime(col: Expr): DateTimeExpr -export function toDateTime(col: Expr): DateTimeExpr -export function toDateTime(col: Expr): Expr { +export function toDateTime(col: Expr): DateTimeExpr +export function toDateTime(col: Expr): DateTimeExpr +export function toDateTime(col: Expr): Expr { // String inputs retain the string flavour; numeric epoch inputs decode to UTC. const input = schemaOf(col) const stringInput = input !== undefined && isStringType(SchemaAST.toType(input.ast)) diff --git a/src/ch/functions/map.ts b/src/ch/functions/map.ts index 19448d2..6dffc0b 100644 --- a/src/ch/functions/map.ts +++ b/src/ch/functions/map.ts @@ -2,7 +2,7 @@ import { makeCond, makeExpr } from "../expr" import { raw, str, compile } from "../../sql/sql-fragment" import type { Expr, Condition, ParamsIn } from "../expr" import * as T from "../types" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { lazy } = builtins("clickhouse", "scalar") @@ -11,25 +11,25 @@ const STRING_MAP = T.map(T.string, T.string).schema // Map functions (handwritten — bracket syntax or custom assembly) -export function mapContains(mapExpr: Expr, Q>, key: string): Condition { +export function mapContains(mapExpr: Expr, Q>, key: string): Condition { return makeCond(lazy(() => `mapContains(${compile(mapExpr.toFragment())}, ${compile(str(key))})`)) } -export function mapGet(mapExpr: Expr, Q>, key: string): Expr { +export function mapGet(mapExpr: Expr, Q>, key: string): Expr { return makeExpr(lazy(() => `${compile(mapExpr.toFragment())}[${compile(str(key))}]`), T.string.schema) } -export function mapKeys(mapExpr: Expr, Q>): Expr, Q> { +export function mapKeys(mapExpr: Expr, Q>): Expr, Q | ClickHouseOnly> { return makeExpr(lazy(() => `mapKeys(${compile(mapExpr.toFragment())})`), STRINGS) } -export function mapValues(mapExpr: Expr, Q>): Expr, Q> { +export function mapValues(mapExpr: Expr, Q>): Expr, Q | ClickHouseOnly> { return makeExpr(lazy(() => `mapValues(${compile(mapExpr.toFragment())})`), STRINGS) } export function mapLiteral< Pairs extends ReadonlyArray]> = ReadonlyArray]>, ->(...pairs: Pairs): Expr, ParamsIn> { +>(...pairs: Pairs): Expr, ParamsIn | ClickHouseOnly> { if (pairs.length === 0) return makeExpr(lazy(() => "map()", "map"), STRING_MAP) const args = () => pairs.map(([k, v]) => `${compile(str(k))}, ${compile(v.toFragment())}`).join(", ") return makeExpr(lazy(() => `map(${args()})`), STRING_MAP) diff --git a/src/ch/functions/numeric.ts b/src/ch/functions/numeric.ts index 1cc94e3..b8b9cac 100644 --- a/src/ch/functions/numeric.ts +++ b/src/ch/functions/numeric.ts @@ -1,32 +1,32 @@ import { numericResultSchema, firstTypedNonNull } from "../define-fn" import type { Expr, ParamsIn } from "../expr" import * as T from "../types" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { compileTypedFnCall, defineFn } = builtins("clickhouse", "scalar") // Type conversion (defineFn one-liners) // Inf/NaN and overflowing numeric strings parse successfully and decode as NaN. -export const toFloat64OrZero = (expr: Expr): Expr => +export const toFloat64OrZero = (expr: Expr): Expr => compileTypedFnCall("toFloat64OrZero", T.CHFloatResult, expr) -export const toFloat64 = (expr: Expr): Expr, Q> => +export const toFloat64 = (expr: Expr): Expr, Q | ClickHouseOnly> => compileTypedFnCall("toFloat64", numericResultSchema(expr), expr) export const toUInt16OrZero = defineFn<[Expr], number>("toUInt16OrZero", T.uint16) export const toUInt64 = defineFn<[Expr | Expr], number>("toUInt64", T.uint64) -export const toInt64 = (expr: Expr): Expr, Q> => +export const toInt64 = (expr: Expr): Expr, Q | ClickHouseOnly> => compileTypedFnCall("toInt64", numericResultSchema(expr), expr) // Arithmetic (compileFnCall wrappers for mixed arg types) -export function intDiv(a: Expr, b: number | Expr): Expr { +export function intDiv(a: Expr, b: number | Expr): Expr { return compileTypedFnCall("intDiv", T.int64.schema, a, b) } export function round_( expr: Expr, decimals?: number, -): Expr, Q> { +): Expr, Q | ClickHouseOnly> { return decimals != null ? compileTypedFnCall("round", numericResultSchema(expr), expr, decimals) : compileTypedFnCall("round", numericResultSchema(expr), expr) @@ -37,14 +37,14 @@ export function round_( type Extremum[]> = Extract> extends never ? number | null : number -export function least_[]>(...exprs: Args): Expr, ParamsIn> { +export function least_[]>(...exprs: Args): Expr, ParamsIn | ClickHouseOnly> { return compileTypedFnCall("least", firstTypedNonNull>()(...exprs), ...exprs) } -export function greatest_[]>(...exprs: Args): Expr, ParamsIn> { +export function greatest_[]>(...exprs: Args): Expr, ParamsIn | ClickHouseOnly> { return compileTypedFnCall("greatest", firstTypedNonNull>()(...exprs), ...exprs) } -export function cityHash64[]>(...exprs: Args): Expr> { +export function cityHash64[]>(...exprs: Args): Expr | ClickHouseOnly> { return compileTypedFnCall("cityHash64", T.uint64.schema, ...exprs) } diff --git a/src/ch/functions/string.ts b/src/ch/functions/string.ts index 9b04fd5..707f597 100644 --- a/src/ch/functions/string.ts +++ b/src/ch/functions/string.ts @@ -4,19 +4,19 @@ import { makeCond } from "../expr" import { compile, str } from "../../sql/sql-fragment" import type { Condition, Expr, ParamsIn } from "../expr" import * as T from "../types" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { compileFnCall, compileTypedFnCall, defineFn, lazy } = builtins("clickhouse", "scalar") const portable = builtins("portable", "scalar") // Standard string functions (defineFn one-liners) -const stringResult = (name: string, expr: Expr): Expr, Q> => +const stringResult = (name: string, expr: Expr): Expr, Q | ClickHouseOnly> => compileTypedFnCall(name, (expr.schema && acceptsSqlNull(expr.schema) ? Schema.NullOr(T.string.schema) : T.string.schema) as Schema.Codec, any>, expr) -export const toString_ = (expr: Expr): Expr, Q> => stringResult("toString", expr) +export const toString_ = (expr: Expr): Expr, Q | ClickHouseOnly> => stringResult("toString", expr) export const length_ = defineFn<[Expr], number>("length", T.uint64) export const lower_ = portable.defineFn<[Expr], string>("lower", T.string) export const positionCaseInsensitive = defineFn<[Expr, Expr], number>( @@ -36,7 +36,7 @@ export const left_ = defineFn<[Expr, Expr], string>("left", T.st /** `hex(x)` — the hex rendering of any value's bytes, as a String. The usual * reason to reach for it is making a hash printable. */ -export const hex = (expr: Expr): Expr, Q> => stringResult("hex", expr) +export const hex = (expr: Expr): Expr, Q | ClickHouseOnly> => stringResult("hex", expr) export const domain_ = defineFn<[Expr], string>("domain", T.string) export const path_ = defineFn<[Expr], string>("path", T.string) @@ -44,15 +44,15 @@ export const cutQueryString = defineFn<[Expr], string>("cutQueryString", // Mixed Expr + literal args (compileFnCall wrappers) -export function position_(haystack: Expr, needle: string): Expr { +export function position_(haystack: Expr, needle: string): Expr { return compileTypedFnCall("position", T.uint64.schema, haystack, needle) } -export function extract_(expr: Expr, pattern: string): Expr { +export function extract_(expr: Expr, pattern: string): Expr { return compileTypedFnCall("extract", T.string.schema, expr, pattern) } -export function replaceOne(haystack: Expr, pattern: string, replacement: string): Expr { +export function replaceOne(haystack: Expr, pattern: string, replacement: string): Expr { return compileTypedFnCall("replaceOne", T.string.schema, haystack, pattern, replacement) } @@ -64,18 +64,18 @@ export function replaceOne(haystack: Expr, pattern: string * where a predicate is wanted, so the SQL reads as a condition rather than * `match(…) = 1`. */ -export function match_(haystack: Expr, pattern: string): Expr { +export function match_(haystack: Expr, pattern: string): Expr { return compileTypedFnCall("match", T.uint8.schema, haystack, pattern) } /** `match(haystack, pattern)` as a predicate — see {@link match_}. */ -export function matchCond(haystack: Expr, pattern: string): Condition { +export function matchCond(haystack: Expr, pattern: string): Condition { return makeCond(lazy(() => `match(${compile(haystack.toFragment())}, ${compile(str(pattern))})`)) } // Variadic string functions -export function concat | string>>(...exprs: Args): Expr> { +export function concat | string>>(...exprs: Args): Expr | ClickHouseOnly> { return compileTypedFnCall("concat", T.string.schema, ...exprs) } @@ -92,17 +92,17 @@ export function concat | string>>(...exprs * Needles are literals by design — the multi-search family requires a constant * array, so there is no expression-valued overload to offer. */ -export function multiSearchAnyCaseInsensitive(haystack: Expr, needles: readonly string[]): Condition { +export function multiSearchAnyCaseInsensitive(haystack: Expr, needles: readonly string[]): Condition { const array = needles.map((needle) => compile(str(needle))).join(", ") return makeCond(lazy(() => `multiSearchAnyCaseInsensitive(${compile(haystack.toFragment())}, [${array}])`)) } -export function hasToken(haystack: Expr, token: Expr | string): Condition { +export function hasToken(haystack: Expr, token: Expr | string): Condition { const call = compileFnCall("hasToken", haystack, token) return makeCond(call.toFragment()) } -export function hasAllTokens(haystack: Expr, tokens: Expr | string): Condition { +export function hasAllTokens(haystack: Expr, tokens: Expr | string): Condition { const call = compileFnCall("hasAllTokens", haystack, tokens) return makeCond(call.toFragment()) } diff --git a/src/ch/functions/window.ts b/src/ch/functions/window.ts index ef987ee..e510478 100644 --- a/src/ch/functions/window.ts +++ b/src/ch/functions/window.ts @@ -4,7 +4,7 @@ import { compile } from "../../sql/sql-fragment" import type { Expr, ParamsIn } from "../expr" import { schemaOf } from "../define-fn" import { QueryBuilderError } from "../errors" -import { builtins } from "./builtin" +import { builtins, type ClickHouseOnly } from "./builtin" const { lazy } = builtins("clickhouse", "scalar") const window = builtins("clickhouse", "window") @@ -100,7 +100,7 @@ function renderWindowSpec(spec: WindowSpec): string { return parts.join(" ") } -export function over(expr: Expr, spec: CompiledWindowSpec): Expr { +export function over(expr: Expr, spec: CompiledWindowSpec): Expr { // A window changes which rows feed the value, never how the value decodes. return makeExpr(window.lazy(() => `${compile(expr.toFragment())} OVER (${spec.sql})`), schemaOf(expr)) } @@ -109,7 +109,7 @@ export function lagInFrame( expr: Expr, offset: number | Expr, defaultValue: T | Expr, -): Expr { +): Expr { return makeExpr( lazy(() => `lagInFrame(${compile(expr.toFragment())}, ${compile(toFragment(offset))}, ${compile(toFragment(defaultValue))})`, diff --git a/src/ch/params-propagation.test-d.ts b/src/ch/params-propagation.test-d.ts index 1d482ab..5738cc8 100644 --- a/src/ch/params-propagation.test-d.ts +++ b/src/ch/params-propagation.test-d.ts @@ -242,3 +242,14 @@ CH.compileUnsafe(eitherName, { name: "a" }) CH.compileUnsafe(eitherName, { name: "a", prevName: "b" }) // @ts-expect-error -- `prevName` must be a string CH.compileUnsafe(eitherName, { prevName: 1 }) + +// orderBy callback: a param in any sort key, not only the first, is required. +const sorted = CH.from(Users) + .select("Id") + .orderBy(($) => [ + [$.Name, "asc"], + [$.Age.add(CH.param.int("bias")), "desc"], + ]) +// @ts-expect-error -- `bias` is missing +CH.compileUnsafe(sorted, {}) +CH.compileUnsafe(sorted, { bias: 1 }) diff --git a/src/ch/query.ts b/src/ch/query.ts index 36752b4..352518c 100644 --- a/src/ch/query.ts +++ b/src/ch/query.ts @@ -279,9 +279,9 @@ export interface CHQuery< orderBy(...specs: Array>): CHQuery /** Sort by expressions of the source's columns, selected or not: `orderBy(($) => [[$.CreatedAt, "desc"]])`. */ - orderBy( - fn: ($: JoinedColumnAccessor) => ReadonlyArray, "asc" | "desc"]>, - ): CHQuery + orderBy, "asc" | "desc"]>>( + fn: ($: JoinedColumnAccessor) => O, + ): CHQuery> /** At most `n` rows: a non-negative integer. */ limit(n: RowCount): CHQuery diff --git a/src/ch/soundness.test.ts b/src/ch/soundness.test.ts index 72a438e..2d26eef 100644 --- a/src/ch/soundness.test.ts +++ b/src/ch/soundness.test.ts @@ -179,10 +179,14 @@ describe("aggregates and GROUP BY", () => { }) describe("function sets", () => { + // The type refuses both too (`dialect.test-d.ts`); this is the check behind it, + // for a caller past the types. it("refuses a ClickHouse function on Postgres, and a Postgres one on ClickHouse", () => { + // @ts-expect-error -- count() is a ClickHouse function expect(() => PG.compileUnsafe(CH.from(Users).select(() => ({ n: CH.count() })))).toThrow( /count\(\) is a ClickHouse function/, ) + // @ts-expect-error -- count(*) is a Postgres function expect(() => compileCHUnsafe(CH.from(Users).select(() => ({ n: PG.count() })))).toThrow(/count\(\) is a Postgres function/) expect(PG.compileUnsafe(CH.from(Users).select(() => ({ n: PG.count() }))).sql).toContain("count(*)") }) diff --git a/src/core.ts b/src/core.ts index bb557c5..64cbf9c 100644 --- a/src/core.ts +++ b/src/core.ts @@ -99,12 +99,16 @@ export { // declarations. Without them public, `tsc --declaration` fails with TS4023 / // TS2742 pointing into a hashed internal chunk. export { + type DialectEntry, + type DialectSatisfied, + type DialectsIn, type OptionalParamEntry, type OptionalParams, type ParamEntry, type ParamsIn, type ParamsSatisfied, } from "./ch/expr" +export type { ClickHouseOnly, PostgresOnly } from "./ch/functions/builtin" export { type CHInsert, @@ -141,8 +145,14 @@ export { type CompiledQueryInput, type CompiledQueryRowSchema, type CHWrite, + type Compile, + type CompileUnion, + type CompileUnionUnsafe, + type CompileUnsafe, type InsertCompileOptions, + type QueryCompileOptions, type RowSchemaMismatch, + type UnionCompileOptions, type TenantScope, CompiledQueryDecodeError, CompiledQueryEncodeError, diff --git a/src/database/database.ts b/src/database/database.ts index aaf89d4..f0f42de 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -18,7 +18,7 @@ import type { AnyInsert } from "../ch/insert" import type { CHDelete, CHUpdate } from "../ch/update" import type { CHQuery, NeedsSelect } from "../ch/query" import type { CHUnionQuery, QueryParams } from "../ch/union" -import type { ParamsSatisfied } from "../ch/expr" +import type { DialectSatisfied, ParamsSatisfied } from "../ch/expr" import { DatabaseError, TransactionClosed, @@ -97,8 +97,14 @@ export interface TransactionInfo { */ export class Transaction extends Context.Service()("@maple-dev/effect-orm/Transaction") {} -export interface DatabaseApi { - readonly dialect: Dialect +/** + * A database. `Functions` is its dialect's function set (`"postgres"` for a + * `Database` built with `postgresDialect`): `run` refuses, in the type, a query + * that uses another set's functions. `string`, the `Database` in context, does + * not check the set, only that a query does not mix two. + */ +export interface DatabaseApi { + readonly dialect: Dialect /** * Compile a query for this database's dialect, run it, and decode its rows. * A write returns its RETURNING rows, or none without `returning`. @@ -106,7 +112,7 @@ export interface DatabaseApi { * runs as it is, if it was compiled for this dialect. */ readonly run: ( - query: Q & RunCheck & ParamsSatisfied, Given>, + query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, Functions>, params?: Given, ) => Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError> /** Run a statement and return its rows, decoded through `schema` when given. */ @@ -139,9 +145,9 @@ export interface DatabaseApi { export class Database extends Context.Service()("@maple-dev/effect-orm/Database") {} -export interface FromSqlClientOptions { +export interface FromSqlClientOptions { /** The dialect the database speaks: `postgresDialect` or `clickhouseDialect`. */ - readonly dialect: Dialect + readonly dialect: Dialect /** * Wraps statements run by `execute`. Pass the ClickHouse client's * `asCommand`, whose query path asks for a JSON result a DDL statement does @@ -202,8 +208,11 @@ const committed = (state: OpenState) => state.open = false }) -/** A `Database` over an Effect `SqlClient`. */ -export const fromSqlClient = (sql: SqlClient.SqlClient, options: FromSqlClientOptions): DatabaseApi => { +/** A `Database` over an Effect `SqlClient`, typed with its dialect's function set. */ +export const fromSqlClient = ( + sql: SqlClient.SqlClient, + options: FromSqlClientOptions, +): DatabaseApi => { const { dialect } = options const capabilities = dialect.transactions ?? noTransactions const command = options.command ?? ((effect) => effect) @@ -478,13 +487,13 @@ export const layerSqlClient = (options: FromSqlClientOptions): Layer.Layer(yield* SqlClient.SqlClient, options) }), ) /** `run` on the `Database` in context. */ export const run = ( - query: Q & RunCheck & ParamsSatisfied, Given>, + query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, string>, params?: Given, ): Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError, Database> => Effect.flatMap(Effect.service(Database), (db) => db.run(query, params)) diff --git a/src/pg/dialect.ts b/src/pg/dialect.ts index e76a414..b1aa923 100644 --- a/src/pg/dialect.ts +++ b/src/pg/dialect.ts @@ -85,7 +85,7 @@ const placeholderCasts: Readonly> = { * rather than ClickHouse's `1`/`0`, and `param.dateTime` an ISO-8601 instant * rather than a zoneless string a session time zone could reinterpret. */ -export const postgresDialect: Dialect = { +export const postgresDialect: Dialect<"postgres"> = { name: "postgres", functions: "postgres", quoteIdent, diff --git a/src/pg/functions.ts b/src/pg/functions.ts index 827d894..b4cc0e0 100644 --- a/src/pg/functions.ts +++ b/src/pg/functions.ts @@ -14,7 +14,7 @@ import { schemaOf, withoutNull } from "../ch/define-fn" import { compile, known, str } from "../sql/sql-fragment" import { encodeLiteral } from "../ch/literal" import type { CHType } from "../ch/types" -import { builtins } from "../ch/functions/builtin" +import { builtins, type PostgresOnly } from "../ch/functions/builtin" import * as T from "./types" const sql = (expr: Expr | Condition): string => compile(expr.toFragment()) @@ -28,30 +28,30 @@ const int8 = T.int8.schema as Schema.Codec // Aggregates /** `count(*)`. */ -export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), int8) +export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), int8) /** `count(DISTINCT expr)`. */ -export const countDistinct = (expr: Expr): Expr => +export const countDistinct = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `count(DISTINCT ${sql(expr)})`), int8) /** `count(*) FILTER (WHERE condition)`: ClickHouse's `countIf`. */ -export const countIf = (condition: Condition): Expr => +export const countIf = (condition: Condition): Expr => makeExpr(aggregate.lazy(() => `count(*) FILTER (WHERE ${sql(condition)})`), int8) /** `sum(expr)`. NULL over no rows, and a string for int8/numeric inputs on the * wire, which the result codec reads as a number. */ -export const sum = (expr: Expr): Expr => +export const sum = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `sum(${sql(expr)})`), nullableNumber) /** `sum(expr) FILTER (WHERE condition)`: ClickHouse's `sumIf`. */ export const sumIf = ( expr: Expr, condition: Condition, -): Expr => +): Expr => makeExpr(aggregate.lazy(() => `sum(${sql(expr)}) FILTER (WHERE ${sql(condition)})`), nullableNumber) /** `avg(expr)`. NULL over no rows. */ -export const avg = (expr: Expr): Expr => +export const avg = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `avg(${sql(expr)})`), nullableNumber) const nullableOf = (expr: Expr): Schema.Codec | undefined => { @@ -60,14 +60,14 @@ const nullableOf = (expr: Expr): Schema.Codec | undefin } /** `min(expr)`, decoding as `expr` does. NULL over no rows. */ -export const min = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `min(${sql(expr)})`), nullableOf(expr)) +export const min = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `min(${sql(expr)})`), nullableOf(expr)) /** `max(expr)`, decoding as `expr` does. NULL over no rows. */ -export const max = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `max(${sql(expr)})`), nullableOf(expr)) +export const max = (expr: Expr): Expr => makeExpr(aggregate.lazy(() => `max(${sql(expr)})`), nullableOf(expr)) /** `percentile_cont(fraction) WITHIN GROUP (ORDER BY expr)`: an interpolated * quantile, ClickHouse's `quantileExact` family. */ -export const percentileCont = (fraction: number, expr: Expr): Expr => { +export const percentileCont = (fraction: number, expr: Expr): Expr => { return makeExpr( aggregate.lazy(() => fraction >= 0 && fraction <= 1 @@ -79,7 +79,7 @@ export const percentileCont = (fraction: number, expr: Expr(expr: Expr): Expr | null, Q> => { +export const arrayAgg = (expr: Expr): Expr | null, Q | PostgresOnly> => { const element = schemaOf(expr) return makeExpr( aggregate.lazy(() => `array_agg(${sql(expr)})`), @@ -95,12 +95,12 @@ export type DateTruncUnit = "second" | "minute" | "hour" | "day" | "week" | "mon /** `date_trunc(unit, ts, 'UTC')`: buckets in UTC whatever the session time * zone, as ClickHouse's `toStartOf*` functions do. Postgres 12+. */ -export const dateTrunc = (unit: DateTruncUnit, ts: Expr): Expr => +export const dateTrunc = (unit: DateTruncUnit, ts: Expr): Expr => makeExpr(scalar.lazy(() => `date_trunc(${compile(str(unit))}, ${sql(ts)}, 'UTC')`), timestamptz) /** `date_bin(seconds, ts, epoch)`: fixed-width buckets aligned to the Unix * epoch, ClickHouse's `toStartOfInterval`. Postgres 14+. */ -export const dateBin = (seconds: number, ts: Expr): Expr => { +export const dateBin = (seconds: number, ts: Expr): Expr => { return makeExpr( scalar.lazy(() => Number.isSafeInteger(seconds) && seconds > 0 @@ -112,15 +112,15 @@ export const dateBin = (seconds: number, ts: Expr): } /** `now()`: the transaction's start time. */ -export const now = (): Expr => makeExpr(scalar.lazy(() => "now()", "now"), timestamptz) +export const now = (): Expr => makeExpr(scalar.lazy(() => "now()", "now"), timestamptz) // Strings and values const text = T.text.schema as Schema.Codec -export const lower = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `lower(${sql(expr)})`), text) -export const upper = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `upper(${sql(expr)})`), text) -export const length = (expr: Expr): Expr => +export const lower = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `lower(${sql(expr)})`), text) +export const upper = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `upper(${sql(expr)})`), text) +export const length = (expr: Expr): Expr => makeExpr(scalar.lazy(() => `length(${sql(expr)})`), T.int4.schema as Schema.Codec) /** `coalesce(expr, fallback)`, no longer nullable. */ @@ -131,15 +131,15 @@ export const coalesce = (expr: Expr, fa ) /** `expr ->> key`: a jsonb field as text, NULL when it is absent. */ -export const jsonText = (expr: Expr, key: string): Expr => +export const jsonText = (expr: Expr, key: string): Expr => makeExpr(scalar.lazy(() => `(${sql(expr)} ->> ${compile(str(key))})`), Schema.NullOr(Schema.String) as Schema.Codec) /** `greatest(a, b, ...)`, decoding as `first` does. Postgres skips NULL arguments. */ -export const greatest = (first: Expr, ...rest: ReadonlyArray>): Expr => +export const greatest = (first: Expr, ...rest: ReadonlyArray>): Expr => makeExpr(scalar.lazy(() => `greatest(${[first, ...rest].map(sql).join(", ")})`), schemaOf(first)) /** `least(a, b, ...)`, decoding as `first` does. Postgres skips NULL arguments. */ -export const least = (first: Expr, ...rest: ReadonlyArray>): Expr => +export const least = (first: Expr, ...rest: ReadonlyArray>): Expr => makeExpr(scalar.lazy(() => `least(${[first, ...rest].map(sql).join(", ")})`), schemaOf(first)) /** @@ -149,7 +149,7 @@ export const least = (first: Expr, ...rest: ReadonlyArray( branches: ReadonlyArray, Expr]>, otherwise: Expr, -): Expr => +): Expr => makeExpr( scalar.lazy( () => `CASE ${branches.map(([when, then]) => `WHEN ${sql(when)} THEN ${sql(then)}`).join(" ")} ELSE ${sql(otherwise)} END`, @@ -158,7 +158,7 @@ export const caseWhen = ( ) /** A condition as a boolean value, to select it or `set` a column from it. */ -export const asBoolean = (condition: Condition): Expr => +export const asBoolean = (condition: Condition): Expr => makeExpr(scalar.lazy(() => `(${sql(condition)})`), Schema.Boolean as Schema.Codec) /** diff --git a/src/postgres.ts b/src/postgres.ts index ae0517f..c0bfe43 100644 --- a/src/postgres.ts +++ b/src/postgres.ts @@ -10,6 +10,10 @@ import { compileCHUnsafe, compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH, + type Compile, + type CompileUnion, + type CompileUnionUnsafe, + type CompileUnsafe, } from "./ch/compile" import { activeDialect, type Dialect } from "./ch/dialect" import { postgresDialect } from "./pg/dialect" @@ -67,16 +71,16 @@ const pgOptions = - Effect.suspend(() => compileCH(query, params, pgOptions(options)))) as typeof compileCH + Effect.suspend(() => compileCH(query, params, pgOptions(options)))) as Compile<"postgres"> /** `compileUnsafe` for Postgres. */ export const compileUnsafe = ((query: any, params?: any, options?: any) => - compileCHUnsafe(query, params, pgOptions(options))) as typeof compileCHUnsafe + compileCHUnsafe(query, params, pgOptions(options))) as CompileUnsafe<"postgres"> /** `compileUnion` for Postgres. */ -export const compileUnion: typeof compileUnionCH = (union, params, options) => - Effect.suspend(() => compileUnionCH(union, params, pgOptions(options))) +export const compileUnion: CompileUnion<"postgres"> = (union, params, options) => + Effect.suspend(() => compileUnionCH(union as any, params, pgOptions(options))) /** `compileUnionUnsafe` for Postgres. */ -export const compileUnionUnsafe: typeof compileUnionUnsafeCH = (union, params, options) => - compileUnionUnsafeCH(union, params, pgOptions(options)) +export const compileUnionUnsafe: CompileUnionUnsafe<"postgres"> = (union, params, options) => + compileUnionUnsafeCH(union as any, params, pgOptions(options)) From 874cf3483543ef9f1db674cc712fe4d717b258f8 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:14:23 +0200 Subject: [PATCH 4/8] feat(types): refuse a column of a source the query does not have A column ref carries a SourceEntry for its source: the FROM alias or table name, or the join alias. Each query removes the entries of its own sources, so an entry left on a query names a source it is missing. compile, compileUnion and run refuse it in the type. A correlated subquery's outer column is provided by the outer query that embeds it. --- src/ch/compile.ts | 23 ++++-- src/ch/expr.ts | 44 ++++++++++-- src/ch/query.ts | 152 +++++++++++++++++++++++++-------------- src/ch/sources.test-d.ts | 94 ++++++++++++++++++++++++ src/core.ts | 4 ++ src/database/database.ts | 6 +- 6 files changed, 253 insertions(+), 70 deletions(-) create mode 100644 src/ch/sources.test-d.ts diff --git a/src/ch/compile.ts b/src/ch/compile.ts index ed3d3ec..7571b85 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -15,7 +15,16 @@ import { isInsert, type AnyInsert, type CHInsert } from "./insert" import { isDelete, isUpdate, type CHDelete, type CHUpdate } from "./update" import type { Table } from "./table" import { createColumnAccessor, createQualifiedColumnAccessor, createJoinedColumnAccessor, sourceAlias } from "./query" -import { aliased, columnTypeOf, isExprLike, type Condition, type DialectSatisfied, type Expr, type ParamsSatisfied } from "./expr" +import { + aliased, + columnTypeOf, + isExprLike, + type Condition, + type DialectSatisfied, + type Expr, + type ParamsSatisfied, + type SourcesSatisfied, +} from "./expr" import { raw, identPath, quoteIdent, quoteIdentPath, compile as compileSqlFragment, type SqlFragment } from "../sql/sql-fragment" import { splitTerminalClauses } from "../sql/terminal-clauses" import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder" @@ -606,7 +615,7 @@ function compileQueryOrWrite( export interface Compile { /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ ( - insert: CHWrite & ParamsSatisfied & DialectSatisfied, + insert: CHWrite & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, params?: Given, options?: InsertCompileOptions, ): Effect.Effect, QueryBuilderError> @@ -623,7 +632,7 @@ export interface Compile { query: CHQuery & NeedsSelect & ParamsSatisfied & - DialectSatisfied, + DialectSatisfied & SourcesSatisfied, /** Values for the query's `param.*` markers: required, and typed, when it has any. */ params?: Given, options?: QueryCompileOptions, @@ -634,7 +643,7 @@ export interface Compile { export interface CompileUnsafe { /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ ( - insert: CHWrite & ParamsSatisfied & DialectSatisfied, + insert: CHWrite & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, params?: Given, options?: InsertCompileOptions, ): CompiledQuery @@ -651,7 +660,7 @@ export interface CompileUnsafe { query: CHQuery & NeedsSelect & ParamsSatisfied & - DialectSatisfied, + DialectSatisfied & SourcesSatisfied, /** Values for the query's `param.*` markers: required, and typed, when it has any. */ params?: Given, options?: QueryCompileOptions, @@ -699,7 +708,7 @@ export type CompileUnion = < const Given extends object = {}, Functions extends string = Default, >( - union: CHUnionQuery & ParamsSatisfied & DialectSatisfied, + union: CHUnionQuery & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, params?: Given, options?: UnionCompileOptions, ) => Effect.Effect, QueryBuilderError> @@ -711,7 +720,7 @@ export type CompileUnionUnsafe = < const Given extends object = {}, Functions extends string = Default, >( - union: CHUnionQuery & ParamsSatisfied & DialectSatisfied, + union: CHUnionQuery & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, params?: Given, options?: UnionCompileOptions, ) => CompiledQuery diff --git a/src/ch/expr.ts b/src/ch/expr.ts index caacb1f..c16eaee 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -101,7 +101,11 @@ export type ParamEntries = IsUnion extends tru type IsUnion = [T] extends [never] ? false : T extends unknown ? ([U] extends [T] ? false : true) : never /** `P` with every entry made optional. */ -export type OptionalParams

= P extends ParamEntry ? OptionalParamEntry : P extends DialectEntry ? P : never +export type OptionalParams

= P extends ParamEntry + ? OptionalParamEntry + : P extends DialectEntry | SourceEntry + ? P + : never /** The names some entry of `P` requires. */ type RequiredNames

= P extends ParamEntry ? (P extends { readonly optional: true } ? never : P["name"]) : never @@ -118,13 +122,41 @@ export interface DialectEntry { readonly functions: Set } -/** The params of an expression, condition, or a union or array of them, and its dialect entries. */ +/** + * The row source a column ref reads from, carried beside its params: `$.Name` + * in `from(Events, "e")` has a `SourceEntry<"e">`. Each query removes the + * entries of its own FROM and JOIN sources from what it carries, so an entry + * left on a query names a source the query does not have. That is a column + * of another query's accessor, used outside the correlated subquery it belongs + * to; `compile` and `run` refuse it (see `SourcesSatisfied`). + */ +export interface SourceEntry { + readonly source: Name +} + +/** The `SourceEntry` of a ref to source `Name`: none for `never`, a ref that names no source. */ +export type SourceOf = [Name] extends [never] ? never : SourceEntry + +/** + * `unknown` when the entries `P` read no source the query is missing; + * otherwise a property naming the source. An entry for `string` (a source + * whose name the type does not know) is not checked. + */ +export type SourcesSatisfied

= [MissingSources

] extends [never] + ? unknown + : { + readonly sourceNotInQuery: `a column of \`${MissingSources

}\` is used, but no FROM or JOIN of the query is named \`${MissingSources

}\`` + } + +type MissingSources

= P extends SourceEntry ? (string extends Name ? never : Name) : never + +/** The params of an expression, condition, or a union or array of them, with its dialect and source entries. */ export type ParamsIn = 0 extends 1 & X ? never : X extends { readonly _params?: (entries: infer P) => void } ? 0 extends 1 & P ? never - : [P] extends [ParamEntry | DialectEntry] + : [P] extends [ParamEntry | DialectEntry | SourceEntry] ? P : never : never @@ -295,8 +327,10 @@ export type Quotient = [R] extends [number] : number | Extract : number | null -export interface ColumnRef> extends Expr< - InferTS +/** A column of a query's source. `Source` is the source's name in the query (see `SourceEntry`). */ +export interface ColumnRef, Source extends string = never> extends Expr< + InferTS, + SourceOf > { readonly columnName: Name /** diff --git a/src/ch/query.ts b/src/ch/query.ts index 352518c..cc2070c 100644 --- a/src/ch/query.ts +++ b/src/ch/query.ts @@ -26,7 +26,7 @@ import type { ColumnDefs, CHType, InferTS, OutputToColumnDefs, NullableColumnDefs } from "./types" import type { Table } from "./table" -import type { Expr, Condition, ColumnRef, ParamsIn, Widen } from "./expr" +import type { Expr, Condition, ColumnRef, ParamsIn, SourceEntry, Widen } from "./expr" import { makeColumnRef } from "./expr" import type { TenantScope } from "./compile" @@ -51,18 +51,34 @@ const selectEvery = ($: any): Record => Object.fromEntries(columns.map((column) => [column, $[column]])) -export type ColumnAccessor = { - readonly [K in keyof Cols & string]: ColumnRef +/** The columns of one source. `Source` is its name in the query, which each ref carries (see `SourceEntry`). */ +export type ColumnAccessor = { + readonly [K in keyof Cols & string]: ColumnRef } /** Combined accessor: main table columns + nested alias accessors for joins. */ export type JoinedColumnAccessor< Cols extends ColumnDefs, Joins extends Record, -> = ColumnAccessor & { - readonly [A in keyof Joins & string]: ColumnAccessor + Source extends string = never, +> = ColumnAccessor & { + readonly [A in keyof Joins & string]: ColumnAccessor } +/** + * `P` without the source entries of `Sources`: what a query carries on from a + * callback over its own sources. What is left names a source it does not have. + */ +export type BindSources = Exclude> + +/** The sources a query provides: its FROM source and its join aliases. */ +type OwnSources = Source | (keyof Joins & string) + +/** A query's entries after a join under `Alias`: the join provides `Alias`, also to refs used before it. */ +type JoinParams = + | BindSources + | BindSources | Alias> + type SelectRecord = Record> /** @@ -99,10 +115,13 @@ export interface LockClause extends LockOptions { } /** Callback for ON conditions — receives main and joined column accessors. */ -export type JoinOnCallback = ( - main: ColumnAccessor, - joined: ColumnAccessor, -) => Condition

+export type JoinOnCallback< + MainCols extends ColumnDefs, + JoinedCols extends ColumnDefs, + P = never, + MainSource extends string = never, + JoinedSource extends string = never, +> = (main: ColumnAccessor, joined: ColumnAccessor) => Condition

// Query state (runtime storage) @@ -114,7 +133,7 @@ interface TypedJoinClause { readonly innerQuery?: CHQuery readonly alias: string /** ON callback, evaluated during compilation with source codecs. Omitted for CROSS JOIN. */ - readonly on?: JoinOnCallback + readonly on?: JoinOnCallback /** The joined table's tenant column, if it declared one. */ readonly tenantColumn?: string /** The joined table's column definitions, for decoding joined selections. */ @@ -229,36 +248,50 @@ export interface CHQuery< Output extends Record = {}, Joins extends Record = {}, Route extends string | undefined = string | undefined, - /** The `param.*` placeholders the query uses, as `ParamEntry`s. */ + /** The `param.*` placeholders the query uses, as `ParamEntry`s, and its dialect and missing-source entries. */ Params = never, + /** + * The FROM source's name: the alias, else the table name. Its refs carry it + * (see `SourceEntry`). `never`, the default, is a query type written without + * one, which takes a query of any source. + */ + Source extends string = never, > { /** @internal — runtime query state */ readonly _state: CHQueryState /** phantom */ - readonly _phantom?: { cols: Cols; output: Output; joins: Joins; route: Route; params: (entries: Params) => void } + readonly _phantom?: { + cols: Cols + output: Output + joins: Joins + route: Route + params: (entries: Params) => void + /** Contravariant like `params`: a query of any source is a `CHQuery` written without one. */ + source: (name: Source) => void + } /** Select every column of the FROM table, as drizzle's bare `select()` does. Output keys are the column keys. */ - select(): CHQuery }, Joins, Route, Params> + select(): CHQuery }, Joins, Route, Params, Source> /** Select specific columns by name. Output keys match column names. */ select( ...columns: K[] - ): CHQuery }, Joins, Route, Params> + ): CHQuery }, Joins, Route, Params, Source> /** Select computed expressions via callback. */ select( - fn: ($: JoinedColumnAccessor) => S, - ): CHQuery, Joins, Route, Params | ParamsIn> + fn: ($: JoinedColumnAccessor) => S, + ): CHQuery, Joins, Route, Params | BindSources, OwnSources>, Source> /** * Filter rows: conditions AND-joined, an `undefined` one skipped. Calling it * again adds conditions, ANDed with the earlier ones. */ where>( - fn: ($: JoinedColumnAccessor) => C, - ): CHQuery> + fn: ($: JoinedColumnAccessor) => C, + ): CHQuery, OwnSources>, Source> - groupBy(...keys: Array): CHQuery + groupBy(...keys: Array): CHQuery /** * Post-aggregation filter, applied after `GROUP BY`. @@ -274,51 +307,51 @@ export interface CHQuery< * Calling it again adds conditions, as `where` does. */ having>( - fn: ($: JoinedColumnAccessor) => C, - ): CHQuery> + fn: ($: JoinedColumnAccessor) => C, + ): CHQuery, OwnSources>, Source> - orderBy(...specs: Array>): CHQuery + orderBy(...specs: Array>): CHQuery /** Sort by expressions of the source's columns, selected or not: `orderBy(($) => [[$.CreatedAt, "desc"]])`. */ orderBy, "asc" | "desc"]>>( - fn: ($: JoinedColumnAccessor) => O, - ): CHQuery> + fn: ($: JoinedColumnAccessor) => O, + ): CHQuery, OwnSources>, Source> /** At most `n` rows: a non-negative integer. */ - limit(n: RowCount): CHQuery + limit(n: RowCount): CHQuery /** Skip `n` rows: a non-negative integer. */ - offset(n: RowCount): CHQuery + offset(n: RowCount): CHQuery - format(fmt: "JSON" | "JSONEachRow"): CHQuery + format(fmt: "JSON" | "JSONEachRow"): CHQuery /** `SELECT DISTINCT`: drop duplicate output rows. */ - distinct(): CHQuery + distinct(): CHQuery /** * `SELECT DISTINCT ON (keys)`: keep the first row of each group of these * output aliases, in ORDER BY order (Postgres wants the keys to lead the * ORDER BY). Replaces `distinct()`. */ - distinctOn(...keys: [keyof Output & string, ...Array]): CHQuery + distinctOn(...keys: [keyof Output & string, ...Array]): CHQuery /** * `FOR UPDATE`: lock the selected rows until the transaction ends. Run it * inside `Database.transaction`. Postgres only; replaces any earlier lock. */ - forUpdate(options?: LockOptions): CHQuery + forUpdate(options?: LockOptions): CHQuery /** `FOR NO KEY UPDATE`: as `forUpdate`, without blocking inserts that reference the rows. */ - forNoKeyUpdate(options?: LockOptions): CHQuery + forNoKeyUpdate(options?: LockOptions): CHQuery /** `FOR SHARE`: a shared lock, which blocks writers but not other sharers. */ - forShare(options?: LockOptions): CHQuery + forShare(options?: LockOptions): CHQuery /** `FOR KEY SHARE`: the weakest lock, blocking only deletes and key updates. */ - forKeyShare(options?: LockOptions): CHQuery + forKeyShare(options?: LockOptions): CHQuery /** * Tag this query with an execution route, carried through to the compiled * query as a type-level fact. The tag is opaque to the builder: what routes * exist, and what an executor does with one, is the caller's vocabulary. */ - route(route: NewRoute): CHQuery + route(route: NewRoute): CHQuery /** * Declare that this query deliberately reads across every tenant, forcing @@ -329,26 +362,33 @@ export interface CHQuery< * "someone forgot the tenant filter" until an author says which. Executors * are expected to refuse these on the ordinary read path. */ - crossTenant(): CHQuery + crossTenant(): CHQuery // Type-safe joins with Table innerJoin( table: Table, alias: FreshAlias, - on: JoinOnCallback, - ): CHQuery + on: JoinOnCallback, + ): CHQuery, Source> leftJoin( table: Table, alias: FreshAlias, - on: JoinOnCallback, - ): CHQuery }, Route, Params | OnParams> + on: JoinOnCallback, + ): CHQuery< + Cols, + Output, + Joins & { readonly [K in Alias]: NullableColumnDefs }, + Route, + JoinParams, + Source + > crossJoin( table: Table, alias: FreshAlias, - ): CHQuery + ): CHQuery, Source> // Type-safe joins with subquery (CHQuery) @@ -362,13 +402,14 @@ export interface CHQuery< >( query: CHQuery & NeedsSelect, alias: FreshAlias, - on: JoinOnCallback, OnParams>, + on: JoinOnCallback, OnParams, Source, Alias>, ): CHQuery< Cols, Output, Joins & { readonly [K in Alias]: OutputToColumnDefs }, Route, - Params | JParams | OnParams + JoinParams | JParams, + Source > leftJoinQuery< @@ -381,13 +422,14 @@ export interface CHQuery< >( query: CHQuery & NeedsSelect, alias: FreshAlias, - on: JoinOnCallback, OnParams>, + on: JoinOnCallback, OnParams, Source, Alias>, ): CHQuery< Cols, Output, Joins & { readonly [K in Alias]: NullableColumnDefs> }, Route, - Params | JParams | OnParams + JoinParams | JParams, + Source > crossJoinQuery< @@ -399,7 +441,7 @@ export interface CHQuery< >( query: CHQuery & NeedsSelect, alias: FreshAlias, - ): CHQuery }, Route, Params | JParams> + ): CHQuery }, Route, BindSources | JParams, Source> /** * Add a CTE (WITH clause). The CTE is prepended to the compiled query, and @@ -413,7 +455,7 @@ export interface CHQuery< withCTE, CTEParams = never>( name: string, query: CHQuery & NeedsSelect, - ): CHQuery + ): CHQuery /** * Attach a CTE from pre-compiled SQL. @@ -428,7 +470,7 @@ export interface CHQuery< name: string, sql: string, options?: { readonly tenantScope?: TenantScope }, - ): CHQuery + ): CHQuery } // Type utilities for extracting output types from queries @@ -487,7 +529,7 @@ export function createQualifiedColumnAccessor( alias: string, tenantColumn?: string, columns?: ColumnDefs, -): ColumnAccessor { +): ColumnAccessor { const cache = new Map>>() const get = (prop: string) => { let ref = cache.get(prop) @@ -498,7 +540,7 @@ export function createQualifiedColumnAccessor( return ref } - return new Proxy>({} as ColumnAccessor, { + return new Proxy>({} as ColumnAccessor, { get: (_target, prop) => (typeof prop === "string" ? get(prop) : undefined), ...spreadColumns(columns, get), }) @@ -570,7 +612,7 @@ function makeQuery< Output extends Record, Joins extends Record, Route extends string | undefined, ->(state: CHQueryState): CHQuery { +>(state: CHQueryState): CHQuery { return { _state: state, @@ -740,10 +782,10 @@ function makeQuery< // Entry points -export function from( +export function from( table: Table, - alias?: string, -): CHQuery { + alias?: Alias, +): CHQuery { return makeQuery({ tableName: table.name, tableAlias: alias, @@ -777,7 +819,7 @@ export function fromQuery< >( query: CHQuery & NeedsSelect, alias: Alias, -): CHQuery, {}, {}, undefined, InnerParams> { +): CHQuery, {}, {}, undefined, InnerParams, Alias> { return makeQuery({ tableName: alias, columns: {}, @@ -811,7 +853,7 @@ export function fromQuery< export function fromUnion, Alias extends string, UnionParams = never>( union: import("./union").CHUnionQuery, alias: Alias, -): CHQuery, {}, {}, undefined, UnionParams> { +): CHQuery, {}, {}, undefined, UnionParams, Alias> { return makeQuery({ tableName: alias, columns: {}, diff --git a/src/ch/sources.test-d.ts b/src/ch/sources.test-d.ts new file mode 100644 index 0000000..6f3e917 --- /dev/null +++ b/src/ch/sources.test-d.ts @@ -0,0 +1,94 @@ +// Type-level tests: a column of a source that is not in the query's FROM or +// JOIN is a type error where the query is compiled or run. + +import { expectTypeOf } from "expect-type" +import * as CH from "./index" +import * as Db from "../database" +import type { SourceEntry, SourcesSatisfied } from "./expr" + +const Events = CH.table("events", { Id: CH.string, Name: CH.string, OrgId: CH.string }) +const Services = CH.table("services", { Id: CH.string, Name: CH.string, Team: CH.string }) + +// A column ref names its source; a query discharges its own sources. +CH.from(Events).select(($) => { + expectTypeOf($.Name.eq("a")).toEqualTypeOf>>() + return { name: $.Name } +}) +CH.from(Events, "e").select(($) => { + expectTypeOf($.Name.eq("a")).toEqualTypeOf>>() + return { name: $.Name } +}) +const own = CH.from(Events, "e") + .innerJoin(Services, "s", (e, s) => e.Name.eq(s.Name)) + .select(($) => ({ name: $.Name, team: $.s.Team })) + .where(($) => [$.OrgId.eq("o"), $.s.Id.eq("x")]) + .orderBy(($) => [[$.s.Team, "asc"]]) +expectTypeOf(own).toExtend>() +CH.compileUnsafe(own) + +// A column captured from another query's accessor is not in this one. +let servicesRef!: CH.JoinedColumnAccessor<(typeof Services)["columns"], {}, "s"> +CH.from(Services, "s").select(($) => { + servicesRef = $ + return { id: $.Id } +}) +const leaked = CH.from(Events).select("Id").where(() => [servicesRef.Team.eq("core")]) +// @ts-expect-error -- `s` is not in FROM or JOIN +CH.compileUnsafe(leaked) +// @ts-expect-error -- `s` is not in FROM or JOIN +void CH.compile(leaked) +// @ts-expect-error -- `s` is not in FROM or JOIN +void Db.run(leaked) +const inSelect = CH.from(Events).select(() => ({ team: servicesRef.Team })) +// @ts-expect-error -- `s` is not in FROM or JOIN +CH.compileUnsafe(inSelect) + +// The message names the source. +expectTypeOf>>().toEqualTypeOf<{ + readonly sourceNotInQuery: "a column of `s` is used, but no FROM or JOIN of the query is named `s`" +}>() +expectTypeOf>().toEqualTypeOf() +// A source whose name is not known in the type is not checked. +expectTypeOf>>().toEqualTypeOf() + +// A later JOIN of that name provides it. +const joinedLater = CH.from(Events) + .select("Id") + .where(() => [servicesRef.Team.eq("core")]) + .innerJoin(Services, "s", (e, s) => e.Name.eq(s.Name)) +CH.compileUnsafe(joinedLater) + +// A correlated subquery: the inner query may read the outer source, and the +// outer query provides it. +let outer!: CH.JoinedColumnAccessor<(typeof Services)["columns"], {}, "s"> +const inner = CH.from(Events) + .select(($) => ({ n: $.Name })) + .where(($) => [$.OrgId.eq("o"), $.Name.eq(outer.Name)]) +// @ts-expect-error -- alone, the inner query reads `s` it does not have +CH.compileUnsafe(inner) +const correlated = CH.from(Services, "s") + .select(($) => ({ team: $.Team })) + .where(() => [CH.exists(inner)]) +CH.compileUnsafe(correlated) +// The same inner query under an outer query of another name is refused. +const wrongOuter = CH.from(Services, "svc") + .select(($) => ({ team: $.Team })) + .where(() => [CH.exists(inner)]) +// @ts-expect-error -- `s` is not in FROM or JOIN +CH.compileUnsafe(wrongOuter) + +// Through every way a query nests: FROM, JOIN, CTE and UNION. +// @ts-expect-error -- `s` is not in FROM or JOIN +CH.compileUnsafe(CH.fromQuery(leaked, "sub").select(($) => ({ id: $.Id }))) +// @ts-expect-error -- `s` is not in FROM or JOIN +CH.compileUnsafe(CH.from(Events).innerJoinQuery(leaked, "l", (e, l) => e.Id.eq(l.Id)).select("Id")) +// @ts-expect-error -- `s` is not in FROM or JOIN +CH.compileUnsafe(CH.from(Events).withCTE("x", leaked).select("Id")) +// @ts-expect-error -- `s` is not in FROM or JOIN +CH.compileUnionUnsafe(CH.unionAll(leaked, CH.from(Events).select("Id"))) + +// A subquery's own sources stay its own: they do not leak to the outer query. +const scalar = CH.from(Services).select(($) => ({ n: $.Name })).where(($) => [$.Team.eq("core")]) +CH.compileUnsafe(CH.from(Events).select("Id").where(($) => [CH.inSubquery($.Name, scalar)])) +CH.compileUnsafe(CH.fromQuery(scalar, "sub").select(($) => ({ n: $.n }))) +CH.compileUnionUnsafe(CH.unionAll(scalar, CH.from(Events).select(($) => ({ n: $.Name })))) diff --git a/src/core.ts b/src/core.ts index 64cbf9c..fb79a29 100644 --- a/src/core.ts +++ b/src/core.ts @@ -82,6 +82,7 @@ export { param, paramPlaceholder, type ParamKind, type ParamMarker } from "./ch/ // Query builder export { + type BindSources, type CHQuery, type ColumnAccessor, type JoinedColumnAccessor, @@ -107,6 +108,9 @@ export { type ParamEntry, type ParamsIn, type ParamsSatisfied, + type SourceEntry, + type SourceOf, + type SourcesSatisfied, } from "./ch/expr" export type { ClickHouseOnly, PostgresOnly } from "./ch/functions/builtin" diff --git a/src/database/database.ts b/src/database/database.ts index f0f42de..ccaf34f 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -18,7 +18,7 @@ import type { AnyInsert } from "../ch/insert" import type { CHDelete, CHUpdate } from "../ch/update" import type { CHQuery, NeedsSelect } from "../ch/query" import type { CHUnionQuery, QueryParams } from "../ch/union" -import type { DialectSatisfied, ParamsSatisfied } from "../ch/expr" +import type { DialectSatisfied, ParamsSatisfied, SourcesSatisfied } from "../ch/expr" import { DatabaseError, TransactionClosed, @@ -112,7 +112,7 @@ export interface DatabaseApi { * runs as it is, if it was compiled for this dialect. */ readonly run: ( - query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, Functions>, + query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, Functions> & SourcesSatisfied>, params?: Given, ) => Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError> /** Run a statement and return its rows, decoded through `schema` when given. */ @@ -493,7 +493,7 @@ export const layerSqlClient = (options: FromSqlClientOptions): Layer.Layer( - query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, string>, + query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, string> & SourcesSatisfied>, params?: Given, ): Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError, Database> => Effect.flatMap(Effect.service(Database), (db) => db.run(query, params)) From de36b892896f3759ee8da748bfc413394969293e Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:16:15 +0200 Subject: [PATCH 5/8] docs: describe the dialect and source checks in the type --- .changeset/type-level-dialect.md | 5 +++ design/dialects.md | 8 +++-- docs/database.md | 2 +- docs/joins-and-subqueries.md | 5 +++ docs/params-and-compilation.md | 62 ++++++++++++++++++++++++++++++++ docs/postgres.md | 9 ++--- docs/reference.md | 21 +++++++---- src/docs-examples.test.ts | 17 +++++++++ 8 files changed, 116 insertions(+), 13 deletions(-) create mode 100644 .changeset/type-level-dialect.md diff --git a/.changeset/type-level-dialect.md b/.changeset/type-level-dialect.md new file mode 100644 index 0000000..7f2f38e --- /dev/null +++ b/.changeset/type-level-dialect.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +A query that cannot compile for its target is now a type error, not only a failure at compile time. A built-in function of one dialect (`CH.countIf`, `PG.countIf`, ...) marks its result with `ClickHouseOnly` or `PostgresOnly`, and `compile`, `compileUnion` and `run` refuse a query with a function of another dialect (`wrongDialect`), or with functions of both (`mixedDialects`). The target is `options.dialect`, else the entry's dialect; `fromSqlClient` now returns a `DatabaseApi<"postgres">` or `DatabaseApi<"clickhouse">` whose `run` checks it. A column ref now names its source (`SourceEntry`), and a query refuses a column of a source that is not in its FROM or JOIN (`sourceNotInQuery`); a correlated subquery can use the outer query's accessor. The runtime checks stay. Also, `orderBy(($) => [...])` now carries the params of every sort key, not only the first. diff --git a/design/dialects.md b/design/dialects.md index 4a7ff35..1b625fc 100644 --- a/design/dialects.md +++ b/design/dialects.md @@ -85,7 +85,11 @@ What we deliberately do not copy: - The package is still named for ClickHouse and the root entry still exports the ClickHouse function catalog. Renaming, or a `core` entry without ClickHouse functions, is a packaging decision for a later release. -- The ClickHouse function catalog writes ClickHouse SQL whatever the dialect. Functions could - refuse to compile under another dialect instead of producing SQL that fails at the server. +- The ClickHouse function catalog writes ClickHouse SQL whatever the dialect. A built-in + function now refuses to compile under another dialect, and is a type error there too: its + result carries a `DialectEntry` beside its params, which `compile` and + `DatabaseApi.run` check against the target. The `Database` in context has no + dialect in its type, so it checks only that a query does not mix two. A `CompiledQuery` does + not carry its dialect in the type yet. - MySQL or SQLite would need `?` placeholders (`reuse: false` already exists) and their own types and functions; nothing in the core should need to change. diff --git a/docs/database.md b/docs/database.md index 2e0cccd..c2b3b10 100644 --- a/docs/database.md +++ b/docs/database.md @@ -80,7 +80,7 @@ Calling `withdraw(1, 30)` outside `transfer` does not compile: `requireTransacti | Export | What it is | | --- | --- | -| `fromSqlClient(sql, options)` | A `DatabaseApi` over a client. `options.dialect` is required; `options.requireTenantScope` refuses unscoped queries ([Tenant scoping](./tenant-scoping.md#enforcing-it)) | +| `fromSqlClient(sql, options)` | A `DatabaseApi` over a client. `options.dialect` is required, and types it: with `postgresDialect` it is a `DatabaseApi<"postgres">`, whose `run` refuses ClickHouse functions in the type ([Checked in the type](./params-and-compilation.md#checked-in-the-type)); `options.requireTenantScope` refuses unscoped queries ([Tenant scoping](./tenant-scoping.md#enforcing-it)) | | `layerSqlClient(options)` | A `Database` layer over the `SqlClient` in context | | `Database` | The service | | `run(query, params?)` | Compile a query for the database's dialect, run it, decode its rows | diff --git a/docs/joins-and-subqueries.md b/docs/joins-and-subqueries.md index 788e67b..a39b671 100644 --- a/docs/joins-and-subqueries.md +++ b/docs/joins-and-subqueries.md @@ -119,6 +119,11 @@ CH.from(Services, "s") .where(($) => [$.OrgId.eq("org_123"), CH.exists(inner)]) ``` +Or build the inner query inside the outer callback and use the outer accessor: `e.Name.eq($.Name)` +writes `s.Name`, and the type checks that the outer query has a source `s` (see +[Checked in the type](./params-and-compilation.md#checked-in-the-type)). A string `outerRef` is +not checked. + All three also accept a pre-compiled SQL string, for the rare case where that is all you have. > **A subquery never scopes its outer query.** `WHERE x IN (SELECT y FROM t WHERE OrgId = 'a')` diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index f24fb9c..9f783ef 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -59,6 +59,68 @@ says so: `defineFn`, `defineCondFn` and `compileTypedFnCall` from the extending hand-written `makeExpr` does not. A param the type does not see is still checked when compiling, as below. +## Checked in the type + +The same channel carries two more facts, and `compile`, `compileUnion` and `run` check them +before the query runs. + +**The dialect of a function.** A built-in function of one dialect marks its result: +`CH.countIf` with `ClickHouseOnly`, `PG.countIf` with `PostgresOnly`. A query that uses one is +a type error where it is compiled or run for the other dialect: + +```ts +const slow = CH.from(Events).select(($) => ({ n: CH.countIf($.DurationMs.gt(500)) })) + +CH.compile(slow) // ok: ClickHouse is the /clickhouse default +PG.compile(slow) // type error: wrongDialect "this query uses ClickHouse functions; it cannot compile for the postgres dialect" +PG.compile(slow, {}, { dialect: CH.clickhouseDialect }) // ok: options.dialect decides +``` + +The target is `options.dialect` when it is given, else the entry's dialect. A `Database` built +with `fromSqlClient(client, { dialect: postgresDialect })` is a `DatabaseApi<"postgres">`, and +its `run` checks the same way. A dialect typed as a plain `Dialect` (a custom one, or the +`Database` in context) is not checked against one set, but a query with functions of both +dialects is refused everywhere (`mixedDialects`): no dialect can compile it. Portable functions +(`nullIf`, `/clickhouse`'s `lower`) and the shared operators add nothing. `CurrentDialect` is +read at run time, so the type cannot see it: under a `CurrentDialect` of another dialect, pass +`options.dialect` too. The compile-time `QueryBuilderDefect` stays, for a caller past the types. + +**The source of a column.** A column ref names its source: the FROM alias, else the table +name, or the join alias. A query provides its own sources, so a column of a source it does not +have is a type error where it is compiled or run. That is a column of another query's accessor, +kept outside the callback it came from: + +```ts +let services!: CH.JoinedColumnAccessor<(typeof Services)["columns"], {}, "s"> +CH.from(Services, "s").select(($) => ((services = $), { id: $.Id })) + +const leaked = CH.from(Events).select("Id").where(() => [services.Team.eq("core")]) +CH.compile(leaked) // type error: sourceNotInQuery "a column of `s` is used, but no FROM or JOIN of the query is named `s`" +``` + +A subquery may read a source of the query it is nested in. The inner query carries the outer +source on, and the outer query provides it, so a correlated subquery can use the outer +accessor in place of a string `outerRef`: + +```ts +CH.from(Services, "s") + .select(($) => ({ team: $.Team })) + .where(($) => [ + $.OrgId.eq("org_123"), + CH.exists( + CH.from(Events) + .select((e) => ({ n: e.Name })) + .where((e) => [e.OrgId.eq("org_123"), e.Name.eq($.Name)]), + ), + ]) +// ... WHERE EXISTS (SELECT events.Name AS n FROM events WHERE ... AND events.Name = s.Name) +``` + +A source whose name the type does not know (a `Table`, a non-literal alias) is not +checked. Writes are not checked: their accessors do not name a source. + +_(Backed by `src/ch/dialect.test-d.ts`, `src/ch/sources.test-d.ts` and `docs/params-and-compilation.md > A correlated subquery reads the outer accessor`.)_ + ## What each kind accepts The declared kind is checked when the value arrives, so a value of the wrong shape is a diff --git a/docs/postgres.md b/docs/postgres.md index 8964dcb..4100ab0 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -209,10 +209,11 @@ document shape, or to name the bad row in its own error); every other read stays The shared operators (`eq`, `in_`, `like`, `ilike`, `and`, `or`, `not`, arithmetic, `lit`) work unchanged. `/postgres` exports only functions Postgres has, plus `nullIf`, which renders the same on both. The ClickHouse catalog on `/clickhouse` (`quantile`, `toStartOfInterval`, its -`count()`, …) writes ClickHouse SQL, so a query that uses one and is compiled for Postgres is a -`QueryBuilderDefect` naming the function; the Postgres functions above fail the same way on -ClickHouse. A custom `Dialect` opts in with `functions: "clickhouse"` or `"postgres"`; without -it, nothing is checked. +`count()`, …) writes ClickHouse SQL, so a query that uses one is a type error where it is +compiled or run for Postgres, and the Postgres functions above are one on ClickHouse (see +[Checked in the type](./params-and-compilation.md#checked-in-the-type)). Past the types, the +compile still fails with a `QueryBuilderDefect` naming the function. A custom `Dialect` opts in +with `functions: "clickhouse"` or `"postgres"`; without it, nothing is checked. ## Known differences diff --git a/docs/reference.md b/docs/reference.md index 658914b..4c0e327 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -88,7 +88,10 @@ to its own dialect; `options.dialect` overrides it. `Dialect`, `DialectClauses` and `ParamStyle` describe a database: how identifiers and literals are written, how params reach the server, and which clauses exist. Pass one as `options.dialect`, or provide it as `CurrentDialect`, a `Context.Reference` -that `compile` reads when no `options.dialect` is given. `DialectTransactions` (with `IsolationLevel` and `TransactionSettings`) says +that `compile` reads when no `options.dialect` is given. `Compile`, `CompileUnsafe`, +`CompileUnion` and `CompileUnionUnsafe` are the types of the four compile +functions, with `Default` the function set of the entry's dialect; their options are +`QueryCompileOptions`, `InsertCompileOptions` and `UnionCompileOptions`. `DialectTransactions` (with `IsolationLevel` and `TransactionSettings`) says which transactions the database supports; see [Database](./database.md). See [Params and compilation](./params-and-compilation.md#dialects) and [Postgres](./postgres.md). @@ -182,17 +185,23 @@ decoded type of a column), `InferEncoded` (its wire type), `ColumnDefs`, `Nullab **Inference** — `InferOutput`, `InferQueryOutput`, `InferUnionOutput`, `SelectRowOf`, `InsertRow`, `InsertRowOf`, `UpdateSet`, `UpdateSetOf`. -**What `compile` checks** — `NeedsSelect` (the query has a SELECT list) and -`ParamsSatisfied` (the params object fills every required param). A helper generic +**What `compile` checks** — `NeedsSelect` (the query has a SELECT list), +`ParamsSatisfied` (the params object fills every required param), +`DialectSatisfied` (no built-in function of another dialect, and not two +dialects in one query) and `SourcesSatisfied` (no column of a source that is not in the +query's FROM or JOIN). See [Params and compilation](./params-and-compilation.md#checked-in-the-type). A helper generic over a query's output takes `CHQuery<…, Output> & NeedsSelect` so it can pass the query on to `compile`. **Params a query carries** — `ParamEntry` (one `param.*` a query needs, in its type), `OptionalParamEntry` and `OptionalParams

` (the same for a param under `when` / `whenTrue`, or one whose name is a union, which the params object may leave out) and -`ParamsIn` (the params an expression or argument list carries). You rarely write them, but -`tsc --declaration` names them when a package exports a query with a param or a `defineFn` helper, -so they are public. +`ParamsIn` (the params an expression or argument list carries). Beside the params, the same +channel carries `DialectEntry` (a built-in function of one dialect: `ClickHouseOnly` or +`PostgresOnly`; `DialectsIn

` reads the sets) and `SourceEntry` (a column of source +`Name`; `SourceOf` is the entry of a ref, `BindSources` removes the ones a query +provides). You rarely write them, but `tsc --declaration` names them when a package exports a +query with a param or a `defineFn` helper, so they are public. **Everything else** — `Table` (what `from` and the write builders accept; every `table` value is one), `Expr`, `ColumnRef`, `Condition`, `Comparable` (what a value of a type may be compared diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index 0db8255..7466e71 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -607,6 +607,23 @@ describe("docs/params-and-compilation.md", () => { expect(compiled.parameters).toEqual(["org_1"]) }), ) + + it("A correlated subquery reads the outer accessor", () => { + const query = CH.from(Services, "s") + .select(($) => ({ team: $.Team })) + .where(($) => [ + $.OrgId.eq("org_123"), + CH.exists( + CH.from(Events) + .select((e) => ({ n: e.Name })) + .where((e) => [e.OrgId.eq("org_123"), e.Name.eq($.Name)]), + ), + ]) + + const sql = oneLine(compileCHUnsafe(query).sql) + expect(sql).toContain("FROM services AS s") + expect(sql).toContain("EXISTS (SELECT events.Name AS n FROM events WHERE events.OrgId = 'org_123' AND events.Name = s.Name)") + }) }) // decoding-results.md From 5ea247ec36926e84f8a3621283733e2d79dbc929 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:16:49 +0200 Subject: [PATCH 6/8] refactor: wrap the checked compile and run signatures, drop two casts --- src/ch/compile.ts | 26 ++++++++++++++++++++------ src/database/database.ts | 12 ++++++++++-- src/postgres.ts | 4 ++-- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 7571b85..c1d09b5 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -615,7 +615,10 @@ function compileQueryOrWrite( export interface Compile { /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ ( - insert: CHWrite & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, + insert: CHWrite & + ParamsSatisfied & + DialectSatisfied & + SourcesSatisfied, params?: Given, options?: InsertCompileOptions, ): Effect.Effect, QueryBuilderError> @@ -632,7 +635,8 @@ export interface Compile { query: CHQuery & NeedsSelect & ParamsSatisfied & - DialectSatisfied & SourcesSatisfied, + DialectSatisfied & + SourcesSatisfied, /** Values for the query's `param.*` markers: required, and typed, when it has any. */ params?: Given, options?: QueryCompileOptions, @@ -643,7 +647,10 @@ export interface Compile { export interface CompileUnsafe { /** An INSERT, UPDATE or DELETE. `params` fills the `param.*` markers among its values. */ ( - insert: CHWrite & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, + insert: CHWrite & + ParamsSatisfied & + DialectSatisfied & + SourcesSatisfied, params?: Given, options?: InsertCompileOptions, ): CompiledQuery @@ -660,7 +667,8 @@ export interface CompileUnsafe { query: CHQuery & NeedsSelect & ParamsSatisfied & - DialectSatisfied & SourcesSatisfied, + DialectSatisfied & + SourcesSatisfied, /** Values for the query's `param.*` markers: required, and typed, when it has any. */ params?: Given, options?: QueryCompileOptions, @@ -708,7 +716,10 @@ export type CompileUnion = < const Given extends object = {}, Functions extends string = Default, >( - union: CHUnionQuery & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, + union: CHUnionQuery & + ParamsSatisfied & + DialectSatisfied & + SourcesSatisfied, params?: Given, options?: UnionCompileOptions, ) => Effect.Effect, QueryBuilderError> @@ -720,7 +731,10 @@ export type CompileUnionUnsafe = < const Given extends object = {}, Functions extends string = Default, >( - union: CHUnionQuery & ParamsSatisfied & DialectSatisfied & SourcesSatisfied, + union: CHUnionQuery & + ParamsSatisfied & + DialectSatisfied & + SourcesSatisfied, params?: Given, options?: UnionCompileOptions, ) => CompiledQuery diff --git a/src/database/database.ts b/src/database/database.ts index ccaf34f..f34a21f 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -112,7 +112,11 @@ export interface DatabaseApi { * runs as it is, if it was compiled for this dialect. */ readonly run: ( - query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, Functions> & SourcesSatisfied>, + query: Q & + RunCheck & + ParamsSatisfied, Given> & + DialectSatisfied, Functions> & + SourcesSatisfied>, params?: Given, ) => Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError> /** Run a statement and return its rows, decoded through `schema` when given. */ @@ -493,7 +497,11 @@ export const layerSqlClient = (options: FromSqlClientOptions): Layer.Layer( - query: Q & RunCheck & ParamsSatisfied, Given> & DialectSatisfied, string> & SourcesSatisfied>, + query: Q & + RunCheck & + ParamsSatisfied, Given> & + DialectSatisfied, string> & + SourcesSatisfied>, params?: Given, ): Effect.Effect>, DatabaseError | QueryBuilderError | CompiledQueryDecodeError, Database> => Effect.flatMap(Effect.service(Database), (db) => db.run(query, params)) diff --git a/src/postgres.ts b/src/postgres.ts index c0bfe43..913740e 100644 --- a/src/postgres.ts +++ b/src/postgres.ts @@ -79,8 +79,8 @@ export const compileUnsafe = ((query: any, params?: any, options?: any) => /** `compileUnion` for Postgres. */ export const compileUnion: CompileUnion<"postgres"> = (union, params, options) => - Effect.suspend(() => compileUnionCH(union as any, params, pgOptions(options))) + Effect.suspend(() => compileUnionCH(union, params, pgOptions(options))) /** `compileUnionUnsafe` for Postgres. */ export const compileUnionUnsafe: CompileUnionUnsafe<"postgres"> = (union, params, options) => - compileUnionUnsafeCH(union as any, params, pgOptions(options)) + compileUnionUnsafeCH(union, params, pgOptions(options)) From 052c33edbe8ce247d150c8fa96c936a332ee0515 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 2 Nov 2026 16:32:14 +0100 Subject: [PATCH 7/8] fix(sqlite): read CurrentDialect in the /sqlite compile functions The /sqlite entry (added on main after this branch) still chose its default dialect when the call was made and ignored CurrentDialect. It now resolves options.dialect, then CurrentDialect, then sqliteDialect when the Effect runs, like /postgres. D1 can be provided once as CurrentDialect. Co-Authored-By: Claude Opus 5.5 --- docs/params-and-compilation.md | 2 +- docs/reference.md | 4 ++-- docs/sqlite.md | 3 ++- src/ch/dialect.test.ts | 18 ++++++++++++++++++ src/sqlite.ts | 22 +++++++++++++++++----- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index f24fb9c..d30d83b 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -177,7 +177,7 @@ CH.compileUnsafe(query, params, options?) // CompiledQuery, throws | `options.rowSchema` | Effect `Schema` used by `decodeRows` / `decodeFirstRow` | | `options.skipFormat` | Omit a trailing `FORMAT` clause (used internally for subqueries) | | `options.deferParams` | Leave placeholders unresolved, for SQL spliced into an outer compile | -| `options.dialect` | Overrides `CurrentDialect` and the entry's dialect (`clickhouseDialect` from `/clickhouse`, `postgresDialect` from `/postgres`) | +| `options.dialect` | Overrides `CurrentDialect` and the entry's dialect (`clickhouseDialect` from `/clickhouse`, `postgresDialect` from `/postgres`, `sqliteDialect` from `/sqlite`) | Each dialect entry exports its own `compile`, defaulting to that dialect. Unions use `compileUnion(union, params)`. diff --git a/docs/reference.md b/docs/reference.md index 7dc670e..96bef0b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -538,8 +538,8 @@ conditions). ## SQLite and D1 `@maple-dev/effect-orm/sqlite` exports the shared query builder and the four compile functions. -`sqliteDialect` is the default; `d1Dialect` uses the same SQL with a 100-binding limit and no -interactive transactions. See [SQLite and D1](./sqlite.md) for driver wiring and limits. +`sqliteDialect` is the default; a `CurrentDialect` in context, or `options.dialect`, overrides it. +`d1Dialect` uses the same SQL with a 100-binding limit and no interactive transactions. See [SQLite and D1](./sqlite.md) for driver wiring and limits. | Export | Purpose | | --- | --- | diff --git a/docs/sqlite.md b/docs/sqlite.md index 2cc50db..8308909 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -153,7 +153,8 @@ export const d1DatabaseLayer = (db: D1Client.D1ClientConfig["db"]) => ``` Use `d1Dialect` when compiling a query separately as well: -`SQLite.compile(query, params, { dialect: SQLite.d1Dialect })`. A query compiled for `sqlite` +`SQLite.compile(query, params, { dialect: SQLite.d1Dialect })`, or provide it once for an Effect +with `Effect.provideService(SQLite.CurrentDialect, SQLite.d1Dialect)`. A query compiled for `sqlite` cannot later run on a database configured as `d1`; pass the builder to `Db.run` instead. D1 compilation refuses more than [100 bound parameters per statement](https://developers.cloudflare.com/d1/platform/limits/). diff --git a/src/ch/dialect.test.ts b/src/ch/dialect.test.ts index bb821cb..a82dd69 100644 --- a/src/ch/dialect.test.ts +++ b/src/ch/dialect.test.ts @@ -5,6 +5,7 @@ import { compileCHUnsafe, compileUnionUnsafe } from "./compile" import type { Dialect } from "./dialect" import * as CH from "./index" import * as PG from "../postgres" +import * as SQLite from "../sqlite" const numbered: Dialect = { ...CH.clickhouseDialect, @@ -252,6 +253,23 @@ describe("dialect from Effect context", () => { }), ) + it.effect("the /sqlite compile defaults to SQLite, and reads CurrentDialect (D1's binding limit)", () => + Effect.gen(function* () { + const users = SQLite.table("users", { email: SQLite.text }) + const query = SQLite.from(users).select("email").where(($) => [$.email.eq("a@x")]) + expect((yield* SQLite.compile(query)).sql).toContain(`"email" = ?`) + const inContext = yield* SQLite.compile(query).pipe(Effect.provideService(SQLite.CurrentDialect, standard)) + expect(inContext.sql).toContain("email = 'a@x'") + const union = yield* SQLite.compileUnion(SQLite.unionAll(query, query)).pipe(Effect.provideService(SQLite.CurrentDialect, standard)) + expect(union.sql).toContain("email = 'a@x'") + // D1 from context: more than 100 bindings is refused before dispatch. + const wide = SQLite.from(users).select("email").where(($) => [$.email.in_(...Array.from({ length: 101 }, (_, index) => `u${index}`))]) + expect((yield* SQLite.compile(wide)).parameters).toHaveLength(101) + const refused = yield* SQLite.compile(wide).pipe(Effect.provideService(SQLite.CurrentDialect, SQLite.d1Dialect), Effect.flip) + expect(refused.message).toMatch(/100/) + }), + ) + it.effect("compiles for two dialects in concurrent fibers", () => Effect.gen(function* () { const compileIn = (dialect: Dialect) => diff --git a/src/sqlite.ts b/src/sqlite.ts index de5b1f7..5bef632 100644 --- a/src/sqlite.ts +++ b/src/sqlite.ts @@ -5,6 +5,7 @@ // the SQLite dialect. Migrations stay with the caller for now. See // docs/sqlite.md. import { compileCH, compileCHRaw, compileCHUnsafe, compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH } from "./ch/compile" +import { activeDialect, type Dialect } from "./ch/dialect" import { compiled, compiledUnsafe } from "./ch/failure" import { table as queryTable, type Table, type TableOptions } from "./ch/table" import type { ColumnDefs } from "./ch/types" @@ -20,6 +21,7 @@ import { type TableDefinition, } from "./schema/sqlite-define" import { sqliteDialect } from "./sqlite/dialect" +import { Effect } from "effect" export * from "./core" export * from "./sqlite/types" @@ -68,21 +70,31 @@ export function table(name: string, definition: any, options?: TableOptions(options: O) => ({ + ...options, + dialect: options?.dialect ?? activeDialect() ?? sqliteDialect, +}) + +/** + * The shared compiler, for SQLite unless `options.dialect` or `CurrentDialect` + * says otherwise. Pass (or provide) d1Dialect for D1's limits. The dialect is + * read when the Effect runs, from the fiber that runs it. + */ export const compile: typeof compileCH = ( query: Parameters[0], params?: Record, options?: Parameters[2], -) => compiled(() => compileCHRaw(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect })) +) => compiled(() => compileCHRaw(query, params, sqliteOptions(options))) /** Compile immediately, throwing on a failed compile. */ export const compileUnsafe: typeof compileCHUnsafe = ( query: Parameters[0], params?: Record, options?: Parameters[2], -) => compiledUnsafe(() => compileCHRaw(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect })) +) => compiledUnsafe(() => compileCHRaw(query, params, sqliteOptions(options))) export const compileUnion: typeof compileUnionCH = (query, params, options) => - compileUnionCH(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect }) + Effect.suspend(() => compileUnionCH(query, params, sqliteOptions(options))) export const compileUnionUnsafe: typeof compileUnionUnsafeCH = (query, params, options) => - compileUnionUnsafeCH(query, params, { ...options, dialect: options?.dialect ?? sqliteDialect }) + compileUnionUnsafeCH(query, params, sqliteOptions(options)) From c3dfdbec6737a3e37ef2964e2f3849e8c8dc8273 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 2 Nov 2026 16:52:59 +0100 Subject: [PATCH 8/8] feat(types): tag SQLite functions and refuse row locks outside Postgres in the type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main's SQLite and D1 dialects did not exist when the dialect tagging was written. SQLite functions now carry SqliteOnly, sqliteDialect and d1Dialect are Dialect<"sqlite">, and the /sqlite compile functions check against "sqlite", so a SQLite query with a ClickHouse or Postgres function (and the reverse) is a type error, as it already was a compile-time defect. Row locks (#85) ride the same channel as a ClauseEntry: forUpdate and the other locks add RowLock, which DialectSatisfied refuses for any target but Postgres (unsupportedClause), matching the compile's run-time refusal. The mixedDialects message names three sets when there are three. The fuzz registry guard looked for `: Dialect = {`; it now also matches `: Dialect<"…"> = {`, or it found no dialect at all. Co-Authored-By: Claude Opus 5.5 --- .changeset/type-level-dialect.md | 2 +- design/dialects.md | 4 +- docs/database.md | 2 +- docs/params-and-compilation.md | 19 +++++++-- docs/reference.md | 7 ++-- docs/sqlite.md | 7 +++- src/ch/dialect.test-d.ts | 69 ++++++++++++++++++++++++++++++++ src/ch/expr.ts | 64 ++++++++++++++++++++++++----- src/ch/functions/builtin.ts | 2 + src/ch/predicates.test.ts | 2 + src/ch/query.ts | 17 +++++--- src/core.ts | 4 +- src/sqlite.ts | 32 ++++++++------- src/sqlite/dialect.ts | 4 +- src/sqlite/functions.ts | 26 ++++++------ src/sqlite/sqlite.test.ts | 2 + tests/fuzz/registry.fuzz.test.ts | 3 +- 17 files changed, 210 insertions(+), 56 deletions(-) diff --git a/.changeset/type-level-dialect.md b/.changeset/type-level-dialect.md index 7f2f38e..564c0cb 100644 --- a/.changeset/type-level-dialect.md +++ b/.changeset/type-level-dialect.md @@ -2,4 +2,4 @@ "@maple-dev/effect-orm": minor --- -A query that cannot compile for its target is now a type error, not only a failure at compile time. A built-in function of one dialect (`CH.countIf`, `PG.countIf`, ...) marks its result with `ClickHouseOnly` or `PostgresOnly`, and `compile`, `compileUnion` and `run` refuse a query with a function of another dialect (`wrongDialect`), or with functions of both (`mixedDialects`). The target is `options.dialect`, else the entry's dialect; `fromSqlClient` now returns a `DatabaseApi<"postgres">` or `DatabaseApi<"clickhouse">` whose `run` checks it. A column ref now names its source (`SourceEntry`), and a query refuses a column of a source that is not in its FROM or JOIN (`sourceNotInQuery`); a correlated subquery can use the outer query's accessor. The runtime checks stay. Also, `orderBy(($) => [...])` now carries the params of every sort key, not only the first. +A query that cannot compile for its target is now a type error, not only a failure at compile time. A built-in function of one dialect (`CH.countIf`, `PG.countIf`, `SQLite.count`, ...) marks its result with `ClickHouseOnly`, `PostgresOnly` or `SqliteOnly` (SQLite and D1 share one set), and `compile`, `compileUnion` and `run` refuse a query with a function of another dialect (`wrongDialect`), or with functions of two (`mixedDialects`). Row locks (`forUpdate`, `forShare`, ...) are refused the same way for ClickHouse, SQLite and D1 (`unsupportedClause`), as only Postgres writes them. The target is `options.dialect`, else the entry's dialect; `fromSqlClient` now returns a `DatabaseApi<"postgres">`, `DatabaseApi<"clickhouse">` or `DatabaseApi<"sqlite">` whose `run` checks it. A column ref now names its source (`SourceEntry`), and a query refuses a column of a source that is not in its FROM or JOIN (`sourceNotInQuery`); a correlated subquery can use the outer query's accessor. The runtime checks stay. Also, `orderBy(($) => [...])` now carries the params of every sort key, not only the first. diff --git a/design/dialects.md b/design/dialects.md index 1b625fc..a350b1f 100644 --- a/design/dialects.md +++ b/design/dialects.md @@ -90,6 +90,8 @@ What we deliberately do not copy: result carries a `DialectEntry` beside its params, which `compile` and `DatabaseApi.run` check against the target. The `Database` in context has no dialect in its type, so it checks only that a query does not mix two. A `CompiledQuery` does - not carry its dialect in the type yet. + not carry its dialect in the type yet. A clause only some dialects write rides the same + channel as a `ClauseEntry` (the row locks, `RowLock`, Postgres only). `format` and + `distinctOn` could use it too; they are refused only at compile time for now. - MySQL or SQLite would need `?` placeholders (`reuse: false` already exists) and their own types and functions; nothing in the core should need to change. diff --git a/docs/database.md b/docs/database.md index 4480ab0..82e56c1 100644 --- a/docs/database.md +++ b/docs/database.md @@ -80,7 +80,7 @@ Calling `withdraw(1, 30)` outside `transfer` does not compile: `requireTransacti | Export | What it is | | --- | --- | -| `fromSqlClient(sql, options)` | A `DatabaseApi` over a client. `options.dialect` is required, and types it: with `postgresDialect` it is a `DatabaseApi<"postgres">`, whose `run` refuses ClickHouse functions in the type ([Checked in the type](./params-and-compilation.md#checked-in-the-type)); `options.requireTenantScope` refuses unscoped queries ([Tenant scoping](./tenant-scoping.md#enforcing-it)) | +| `fromSqlClient(sql, options)` | A `DatabaseApi` over a client. `options.dialect` is required, and types it: with `postgresDialect` it is a `DatabaseApi<"postgres">`, whose `run` refuses ClickHouse and SQLite functions in the type (`d1Dialect` and `sqliteDialect` give a `DatabaseApi<"sqlite">`, which also refuses row locks) ([Checked in the type](./params-and-compilation.md#checked-in-the-type)); `options.requireTenantScope` refuses unscoped queries ([Tenant scoping](./tenant-scoping.md#enforcing-it)) | | `layerSqlClient(options)` | A `Database` layer over the `SqlClient` in context | | `Database` | The service | | `run(query, params?)` | Compile a query for the database's dialect, run it, decode its rows | diff --git a/docs/params-and-compilation.md b/docs/params-and-compilation.md index 92f4666..dc52757 100644 --- a/docs/params-and-compilation.md +++ b/docs/params-and-compilation.md @@ -61,12 +61,13 @@ compiling, as below. ## Checked in the type -The same channel carries two more facts, and `compile`, `compileUnion` and `run` check them +The same channel carries three more facts, and `compile`, `compileUnion` and `run` check them before the query runs. **The dialect of a function.** A built-in function of one dialect marks its result: -`CH.countIf` with `ClickHouseOnly`, `PG.countIf` with `PostgresOnly`. A query that uses one is -a type error where it is compiled or run for the other dialect: +`CH.countIf` with `ClickHouseOnly`, `PG.countIf` with `PostgresOnly`, `SQLite.count` with +`SqliteOnly` (SQLite and D1 share the `"sqlite"` set). A query that uses one is a type error +where it is compiled or run for another dialect: ```ts const slow = CH.from(Events).select(($) => ({ n: CH.countIf($.DurationMs.gt(500)) })) @@ -80,11 +81,21 @@ The target is `options.dialect` when it is given, else the entry's dialect. A `D with `fromSqlClient(client, { dialect: postgresDialect })` is a `DatabaseApi<"postgres">`, and its `run` checks the same way. A dialect typed as a plain `Dialect` (a custom one, or the `Database` in context) is not checked against one set, but a query with functions of both -dialects is refused everywhere (`mixedDialects`): no dialect can compile it. Portable functions +dialects is refused everywhere (`mixedDialects`): no dialect can compile it. `/sqlite`'s +`compile` checks against `"sqlite"`, and so does a `DatabaseApi<"sqlite">` from +`fromSqlClient(client, { dialect: d1Dialect })`. Portable functions (`nullIf`, `/clickhouse`'s `lower`) and the shared operators add nothing. `CurrentDialect` is read at run time, so the type cannot see it: under a `CurrentDialect` of another dialect, pass `options.dialect` too. The compile-time `QueryBuilderDefect` stays, for a caller past the types. +**A clause only some dialects write.** A row lock (`forUpdate`, `forNoKeyUpdate`, `forShare`, +`forKeyShare`) adds a `RowLock` (`ClauseEntry<"FOR UPDATE", "postgres">`): only Postgres has row +locks, so compiling or running the query for ClickHouse, SQLite or D1 is a type error +(`unsupportedClause "FOR UPDATE has no meaning for the sqlite dialect"`). On SQLite and D1, claim a +row with a conditional update instead (`update(…).set(…).where(($) => [$.status.eq("pending")]).returning(…)`), +or keep the writer single (a Durable Object). As with functions, a dialect typed as a plain `Dialect` +is not checked, and the compile still refuses the lock at run time. + **The source of a column.** A column ref names its source: the FROM alias, else the table name, or the join alias. A query provides its own sources, so a column of a source it does not have is a type error where it is compiled or run. That is a column of another query's accessor, diff --git a/docs/reference.md b/docs/reference.md index ad37d2d..7a49799 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -64,7 +64,7 @@ Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or | `limit(n)` / `offset(n)` | Rounded before emission | | `format(fmt)` | `"JSON"` \| `"JSONEachRow"` | | `distinct()` / `distinctOn(...aliases)` | `SELECT DISTINCT` / `SELECT DISTINCT ON (…)` | -| `forUpdate` / `forNoKeyUpdate` / `forShare` / `forKeyShare` | Postgres row locks; options `LockOptions` (`skipLocked`, `noWait`, `of`) | +| `forUpdate` / `forNoKeyUpdate` / `forShare` / `forKeyShare` | Postgres row locks; options `LockOptions` (`skipLocked`, `noWait`, `of`). Refused in the type when compiled or run for another dialect | | `innerJoin` / `leftJoin` / `crossJoin` | `(table, alias, on?)` | | `innerJoinQuery` / `leftJoinQuery` / `crossJoinQuery` | `(query, alias, on?)` | | `withCTE(name, query)` / `withCTE(name, sql, options?)` | Typed query derives scope; SQL form can declare `options.tenantScope` | @@ -198,8 +198,9 @@ to `compile`. `OptionalParamEntry` and `OptionalParams

` (the same for a param under `when` / `whenTrue`, or one whose name is a union, which the params object may leave out) and `ParamsIn` (the params an expression or argument list carries). Beside the params, the same -channel carries `DialectEntry` (a built-in function of one dialect: `ClickHouseOnly` or -`PostgresOnly`; `DialectsIn

` reads the sets) and `SourceEntry` (a column of source +channel carries `DialectEntry` (a built-in function of one dialect: `ClickHouseOnly`, +`PostgresOnly` or `SqliteOnly`; `DialectsIn

` reads the sets), `ClauseEntry` (a +clause only some dialects write: `RowLock` for the Postgres row locks) and `SourceEntry` (a column of source `Name`; `SourceOf` is the entry of a ref, `BindSources` removes the ones a query provides). You rarely write them, but `tsc --declaration` names them when a package exports a query with a param or a `defineFn` helper, so they are public. diff --git a/docs/sqlite.md b/docs/sqlite.md index 8308909..18491de 100644 --- a/docs/sqlite.md +++ b/docs/sqlite.md @@ -184,7 +184,12 @@ written as decimal text, because SQLite reads some decimals as a neighbouring do fraction compared with an `integer` column (`Amount > 9.75`) compiles as on Postgres. It refuses these unsupported forms at compile time: - `DISTINCT ON`, row locks and `ILIKE`. Use `LIKE`, which on SQLite ignores ASCII case - (`'acme' LIKE 'ACME'` is true, unlike Postgres and ClickHouse). + (`'acme' LIKE 'ACME'` is true, unlike Postgres and ClickHouse). A row lock is refused in the + type as well: `SQLite.compile(q.forUpdate())` and `run` on a SQLite or D1 `Database` do not + typecheck. Claim a row with a conditional `update(…).where(…).returning(…)` instead. A + Postgres or ClickHouse function in a SQLite query, and a SQLite function in a Postgres or + ClickHouse one, are type errors too; see + [Checked in the type](./params-and-compilation.md#checked-in-the-type). - Text containing U+0000. SQLite stores it, but its drivers end text there, so the value would read back cut short; the builder refuses it in literals and bound params. - `ON CONSTRAINT` conflict targets. Name the indexed columns instead. diff --git a/src/ch/dialect.test-d.ts b/src/ch/dialect.test-d.ts index a19141e..c42a261 100644 --- a/src/ch/dialect.test-d.ts +++ b/src/ch/dialect.test-d.ts @@ -6,6 +6,7 @@ import { expectTypeOf } from "expect-type" import type * as SqlClient from "effect/sql/SqlClient" import * as CH from "./index" import * as PG from "../postgres" +import * as SQLite from "../sqlite" import * as Db from "../database" import type { Dialect } from "./dialect" import type { DialectSatisfied, DialectEntry } from "./expr" @@ -121,3 +122,71 @@ void Db.run(pgQuery) void Db.run(mixed) // A typed `Database` still provides the `Database` in context. expectTypeOf(Layer.succeed(Db.Database, pg)).toEqualTypeOf>() + +// SQLite and D1: one function set, "sqlite", for both dialects. +const Items = SQLite.table("items", { id: SQLite.integer, note: SQLite.text }) +expectTypeOf(SQLite.count()).toEqualTypeOf>>() +expectTypeOf(SQLite.sqliteDialect).toEqualTypeOf>() +expectTypeOf(SQLite.d1Dialect).toEqualTypeOf>() +const sqliteQuery = SQLite.from(Items).select(($) => ({ n: SQLite.count(), note: SQLite.lower($.note) })) +const sqlitePortable = SQLite.from(Items).select(($) => ({ note: CH.nullIf($.note, "") })) +SQLite.compileUnsafe(sqliteQuery) +SQLite.compileUnsafe(sqliteQuery, {}, { dialect: SQLite.d1Dialect }) +void SQLite.compile(sqliteQuery) +SQLite.compileUnsafe(sqlitePortable) +// @ts-expect-error -- countIf() is a ClickHouse function +SQLite.compileUnsafe(chQuery) +// @ts-expect-error -- countIf(*) FILTER is a Postgres function +SQLite.compileUnsafe(pgQuery, {}, { dialect: SQLite.d1Dialect }) +// @ts-expect-error -- count(*) is a SQLite function +CH.compileUnsafe(sqliteQuery) +// @ts-expect-error -- count(*) is a SQLite function +PG.compileUnsafe(sqliteQuery) +// @ts-expect-error -- count(*) is a SQLite function +CH.compileUnsafe(sqliteQuery, {}, { dialect: CH.clickhouseDialect }) +CH.compileUnsafe(sqliteQuery, {}, { dialect: SQLite.d1Dialect }) +const sqliteUnion = SQLite.unionAll(sqliteQuery, SQLite.from(Items).select(($) => ({ n: SQLite.count(), note: $.note }))) +SQLite.compileUnionUnsafe(sqliteUnion) +// @ts-expect-error -- the union uses SQLite functions +PG.compileUnionUnsafe(sqliteUnion) +expectTypeOf, "postgres">>().toEqualTypeOf<{ + readonly wrongDialect: "this query uses SQLite functions; it cannot compile for the postgres dialect" +}>() +expectTypeOf | DialectEntry<"clickhouse">, string>>().toEqualTypeOf<{ + readonly mixedDialects: "this query uses both ClickHouse and SQLite functions; no dialect can compile it" +}>() +expectTypeOf | DialectEntry<"clickhouse"> | DialectEntry<"postgres">, "sqlite">>().toEqualTypeOf<{ + readonly mixedDialects: "this query uses ClickHouse, Postgres and SQLite functions; no dialect can compile it" +}>() +const d1 = Db.fromSqlClient(client, { dialect: SQLite.d1Dialect }) +expectTypeOf(d1).toEqualTypeOf>() +void d1.run(sqliteQuery) +void d1.run(sqlitePortable) +// @ts-expect-error -- countIf() is a ClickHouse function +void d1.run(chQuery) + +// Row locks: only Postgres writes them, so the type refuses them elsewhere. +const locked = SQLite.from(Items).select("id").forUpdate() +const shared = SQLite.from(Items).select("id").forShare({ skipLocked: true }) +PG.compileUnsafe(locked) +void PG.compile(shared) +void pg.run(locked) +// @ts-expect-error -- FOR UPDATE has no meaning for the sqlite dialect +SQLite.compileUnsafe(locked) +// @ts-expect-error -- FOR SHARE has no meaning for the sqlite dialect (D1 included) +SQLite.compileUnsafe(shared, {}, { dialect: SQLite.d1Dialect }) +// @ts-expect-error -- FOR UPDATE has no meaning for the sqlite dialect +void d1.run(locked) +// @ts-expect-error -- FOR UPDATE has no meaning for the clickhouse dialect +CH.compileUnsafe(locked) +// @ts-expect-error -- FOR KEY SHARE has no meaning for the clickhouse dialect +void CH.compile(CH.from(Events).select("Id").forKeyShare()) +// A lock inside a subquery still counts. +// @ts-expect-error -- FOR NO KEY UPDATE has no meaning for the sqlite dialect +SQLite.compileUnsafe(SQLite.from(Items).select("id").where(() => [SQLite.exists(SQLite.from(Items).select("id").forNoKeyUpdate())])) +// A dialect that names no set is not checked; the compile refuses it at run time if it cannot lock. +CH.compileUnsafe(locked, {}, { dialect: custom }) +expectTypeOf, "sqlite">>().toEqualTypeOf<{ + readonly unsupportedClause: "FOR UPDATE has no meaning for the sqlite dialect" +}>() +expectTypeOf | DialectEntry<"postgres">, "postgres">>().toEqualTypeOf() diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 15a04c1..359a4e4 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -103,7 +103,7 @@ type IsUnion = [T] extends [never] ? false : T extends unknown ? ([U] /** `P` with every entry made optional. */ export type OptionalParams

= P extends ParamEntry ? OptionalParamEntry - : P extends DialectEntry | SourceEntry + : P extends DialectEntry | SourceEntry | ClauseEntry ? P : never @@ -156,39 +156,83 @@ export type ParamsIn = 0 extends 1 & X : X extends { readonly _params?: (entries: infer P) => void } ? 0 extends 1 & P ? never - : [P] extends [ParamEntry | DialectEntry | SourceEntry] + : [P] extends [ParamEntry | DialectEntry | SourceEntry | ClauseEntry] ? P : never : never +/** + * A clause only the dialects whose function set is in `Sets` can write, carried + * beside a query's params: `forUpdate()` adds `ClauseEntry<"FOR UPDATE", "postgres">`, + * as ClickHouse, SQLite and D1 have no row locks. `DialectSatisfied` refuses it + * for another target, as the compile refuses it at run time. + */ +export interface ClauseEntry { + readonly clause: Clause + readonly sets: Sets +} + /** The function sets the entries `P` need: `never` when they need none. */ export type DialectsIn

= P extends DialectEntry ? Set : never -type FunctionSetLabel = Set extends "clickhouse" ? "ClickHouse" : Set extends "postgres" ? "Postgres" : Set +/** The clauses of `P` a dialect of set `Target` cannot write. */ +type UnsupportedClauses = P extends ClauseEntry + ? Target extends Sets + ? never + : Clause + : never + +type FunctionSetLabel = Set extends "clickhouse" + ? "ClickHouse" + : Set extends "postgres" + ? "Postgres" + : Set extends "sqlite" + ? "SQLite" + : Set /** * `unknown` when the entries `P` can compile for a dialect whose function set * is `Target`; otherwise a property saying why. A `Target` of `string` is a * dialect that does not name its set, and is not checked. Functions of two sets - * in one query are refused for every target: no dialect renders both. + * in one query are refused for every target: no dialect renders both. A clause + * the target cannot write (a row lock outside Postgres) is refused as well. */ export type DialectSatisfied = IsUnion> extends true ? { - readonly mixedDialects: `this query uses both ${JoinLabels>} functions; no dialect can compile it` + readonly mixedDialects: `this query uses ${JoinLabels>} functions; no dialect can compile it` } : string extends Target ? unknown : [DialectsIn

] extends [Target] - ? unknown + ? [UnsupportedClauses] extends [never] + ? unknown + : { + readonly unsupportedClause: `${UnsupportedClauses} has no meaning for the ${Target} dialect` + } : { readonly wrongDialect: `this query uses ${FunctionSetLabel>} functions; it cannot compile for the ${Target} dialect` } -/** `"ClickHouse and Postgres"` for a two-set union, in a fixed order. */ -type JoinLabels = [Sets] extends ["clickhouse" | "postgres"] - ? "ClickHouse and Postgres" - : FunctionSetLabel +/** + * `"both ClickHouse and Postgres"`, `"ClickHouse, Postgres and SQLite"`: the + * built-in sets of a union, in a fixed order. A custom set is named as it is. + */ +type JoinLabels = [Exclude] extends [never] + ? JoinWithAnd< + [ + ...("clickhouse" extends Sets ? ["ClickHouse"] : []), + ...("postgres" extends Sets ? ["Postgres"] : []), + ...("sqlite" extends Sets ? ["SQLite"] : []), + ] + > + : `both ${FunctionSetLabel}` + +type JoinWithAnd> = Labels extends readonly [infer A extends string, infer B extends string] + ? `both ${A} and ${B}` + : Labels extends readonly [infer A extends string, infer B extends string, infer C extends string] + ? `${A}, ${B} and ${C}` + : Labels[number] /** The value a param needs: every entry's type for that name, intersected * per entry (a `boolean` entry stays `boolean`, not `true & false`). */ diff --git a/src/ch/functions/builtin.ts b/src/ch/functions/builtin.ts index 724861f..28128a6 100644 --- a/src/ch/functions/builtin.ts +++ b/src/ch/functions/builtin.ts @@ -30,6 +30,8 @@ export type SetEntry = Set extends Functio export type ClickHouseOnly = DialectEntry<"clickhouse"> /** What a Postgres function adds to its result's params: see `DialectEntry`. */ export type PostgresOnly = DialectEntry<"postgres"> +/** What a SQLite function adds to its result's params: see `DialectEntry`. */ +export type SqliteOnly = DialectEntry<"sqlite"> const setLabel: Record = { sqlite: "a SQLite function (from @maple-dev/effect-orm/sqlite)", diff --git a/src/ch/predicates.test.ts b/src/ch/predicates.test.ts index 515f721..e919953 100644 --- a/src/ch/predicates.test.ts +++ b/src/ch/predicates.test.ts @@ -109,6 +109,8 @@ describe("row locking", () => { const defect = function* (q: Effect.Effect) { expect(failure(yield* Effect.exit(q))).toBeInstanceOf(QueryBuilderDefect) } + // The type refuses it too (see `RowLock`); this is the run-time net. + // @ts-expect-error -- FOR UPDATE has no meaning for the clickhouse dialect yield* defect(CH.compile(base.forUpdate(), {})) yield* defect(PG.compile(base.forUpdate({ skipLocked: true, noWait: true }), {})) yield* defect(PG.compile(base.forUpdate({ of: ["public.jobs"] }), {})) diff --git a/src/ch/query.ts b/src/ch/query.ts index cc2070c..af01927 100644 --- a/src/ch/query.ts +++ b/src/ch/query.ts @@ -26,7 +26,7 @@ import type { ColumnDefs, CHType, InferTS, OutputToColumnDefs, NullableColumnDefs } from "./types" import type { Table } from "./table" -import type { Expr, Condition, ColumnRef, ParamsIn, SourceEntry, Widen } from "./expr" +import type { ClauseEntry, Expr, Condition, ColumnRef, ParamsIn, SourceEntry, Widen } from "./expr" import { makeColumnRef } from "./expr" import type { TenantScope } from "./compile" @@ -96,6 +96,13 @@ export type InferOutput = { type OrderBySpec = [keyof Output & string, "asc" | "desc"] +/** + * What a row lock adds to a query's params: only Postgres writes it, so + * `compile` and `run` for ClickHouse, SQLite or D1 refuse the query in the + * type (see `DialectSatisfied`), as the compile does at run time. + */ +export type RowLock = ClauseEntry + /** How a locking clause waits for rows another transaction holds. */ export interface LockOptions { /** `SKIP LOCKED`: leave out rows another transaction has locked. */ @@ -338,13 +345,13 @@ export interface CHQuery< * `FOR UPDATE`: lock the selected rows until the transaction ends. Run it * inside `Database.transaction`. Postgres only; replaces any earlier lock. */ - forUpdate(options?: LockOptions): CHQuery + forUpdate(options?: LockOptions): CHQuery, Source> /** `FOR NO KEY UPDATE`: as `forUpdate`, without blocking inserts that reference the rows. */ - forNoKeyUpdate(options?: LockOptions): CHQuery + forNoKeyUpdate(options?: LockOptions): CHQuery, Source> /** `FOR SHARE`: a shared lock, which blocks writers but not other sharers. */ - forShare(options?: LockOptions): CHQuery + forShare(options?: LockOptions): CHQuery, Source> /** `FOR KEY SHARE`: the weakest lock, blocking only deletes and key updates. */ - forKeyShare(options?: LockOptions): CHQuery + forKeyShare(options?: LockOptions): CHQuery, Source> /** * Tag this query with an execution route, carried through to the compiled diff --git a/src/core.ts b/src/core.ts index fb79a29..37afd38 100644 --- a/src/core.ts +++ b/src/core.ts @@ -90,6 +90,7 @@ export { type InferOutput, type InferQueryOutput, type LockOptions, + type RowLock, // What `compile` asks of a query, for a wrapper generic over its output. type NeedsSelect, from, @@ -100,6 +101,7 @@ export { // declarations. Without them public, `tsc --declaration` fails with TS4023 / // TS2742 pointing into a hashed internal chunk. export { + type ClauseEntry, type DialectEntry, type DialectSatisfied, type DialectsIn, @@ -112,7 +114,7 @@ export { type SourceOf, type SourcesSatisfied, } from "./ch/expr" -export type { ClickHouseOnly, PostgresOnly } from "./ch/functions/builtin" +export type { ClickHouseOnly, PostgresOnly, SqliteOnly } from "./ch/functions/builtin" export { type CHInsert, diff --git a/src/sqlite.ts b/src/sqlite.ts index 5bef632..9474ecd 100644 --- a/src/sqlite.ts +++ b/src/sqlite.ts @@ -4,7 +4,15 @@ // DDL, SQLite functions, the query builder and a `compile` that defaults to // the SQLite dialect. Migrations stay with the caller for now. See // docs/sqlite.md. -import { compileCH, compileCHRaw, compileCHUnsafe, compileUnion as compileUnionCH, compileUnionUnsafe as compileUnionUnsafeCH } from "./ch/compile" +import { + compileCHRaw, + compileUnion as compileUnionCH, + compileUnionUnsafe as compileUnionUnsafeCH, + type Compile, + type CompileUnion, + type CompileUnionUnsafe, + type CompileUnsafe, +} from "./ch/compile" import { activeDialect, type Dialect } from "./ch/dialect" import { compiled, compiledUnsafe } from "./ch/failure" import { table as queryTable, type Table, type TableOptions } from "./ch/table" @@ -79,22 +87,18 @@ const sqliteOptions = [0], - params?: Record, - options?: Parameters[2], -) => compiled(() => compileCHRaw(query, params, sqliteOptions(options))) +// Typed through `any` and cast, as in `/postgres`: `Compile` is overloaded. +export const compile = ((query: any, params?: any, options?: any) => + compiled(() => compileCHRaw(query, params, sqliteOptions(options)))) as Compile<"sqlite"> /** Compile immediately, throwing on a failed compile. */ -export const compileUnsafe: typeof compileCHUnsafe = ( - query: Parameters[0], - params?: Record, - options?: Parameters[2], -) => compiledUnsafe(() => compileCHRaw(query, params, sqliteOptions(options))) +export const compileUnsafe = ((query: any, params?: any, options?: any) => + compiledUnsafe(() => compileCHRaw(query, params, sqliteOptions(options)))) as CompileUnsafe<"sqlite"> -export const compileUnion: typeof compileUnionCH = (query, params, options) => +export const compileUnion: CompileUnion<"sqlite"> = (query, params, options) => Effect.suspend(() => compileUnionCH(query, params, sqliteOptions(options))) -export const compileUnionUnsafe: typeof compileUnionUnsafeCH = (query, params, options) => +export const compileUnionUnsafe: CompileUnionUnsafe<"sqlite"> = (query, params, options) => compileUnionUnsafeCH(query, params, sqliteOptions(options)) diff --git a/src/sqlite/dialect.ts b/src/sqlite/dialect.ts index f4a79f9..b0d9dd5 100644 --- a/src/sqlite/dialect.ts +++ b/src/sqlite/dialect.ts @@ -30,7 +30,7 @@ const literal = (value: unknown, context: string): string => { } /** SQLite with positional bindings and transactions supplied by the caller's SqlClient. */ -export const sqliteDialect: Dialect = { +export const sqliteDialect: Dialect<"sqlite"> = { name: "sqlite", functions: "sqlite", ilike: false, @@ -77,7 +77,7 @@ export const sqliteDialect: Dialect = { } /** D1 executes SQLite SQL, limits each statement to 100 bindings, and has no interactive transactions. */ -export const d1Dialect: Dialect = { +export const d1Dialect: Dialect<"sqlite"> = { ...sqliteDialect, name: "d1", params: { ...sqliteDialect.params, _tag: "bind", placeholder: () => "?", reuse: false, maxParameters: 100 }, diff --git a/src/sqlite/functions.ts b/src/sqlite/functions.ts index afcc9b6..6b3ec97 100644 --- a/src/sqlite/functions.ts +++ b/src/sqlite/functions.ts @@ -1,37 +1,39 @@ import { Schema } from "effect" import { makeExpr, type Condition, type Expr } from "../ch/expr" import { schemaOf } from "../ch/define-fn" -import { builtins } from "../ch/functions/builtin" +import { builtins, type SqliteOnly } from "../ch/functions/builtin" import { compile } from "../sql/sql-fragment" +// Each result carries `SqliteOnly`, as `builtins("sqlite", …)` refuses them for +// another dialect at compile: the type refuses them there too. const aggregate = builtins("sqlite", "aggregate") const scalar = builtins("sqlite", "scalar") const sql = (expr: Expr | Condition): string => compile(expr.toFragment()) const nullableNumber = Schema.NullOr(Schema.Finite) -export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), Schema.Int) -export const countDistinct =

(expr: Expr): Expr => +export const count = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), Schema.Int) +export const countDistinct =

(expr: Expr): Expr => makeExpr(aggregate.lazy(() => `count(DISTINCT ${sql(expr)})`, "count"), Schema.Int) -export const countIf =

(condition: Condition

): Expr => +export const countIf =

(condition: Condition

): Expr => makeExpr(aggregate.lazy(() => `count(*) FILTER (WHERE ${sql(condition)})`, "count"), Schema.Int) -export const sum =

(expr: Expr): Expr => +export const sum =

(expr: Expr): Expr => makeExpr(aggregate.lazy(() => `sum(${sql(expr)})`, "sum"), nullableNumber) -export const avg =

(expr: Expr): Expr => +export const avg =

(expr: Expr): Expr => makeExpr(aggregate.lazy(() => `avg(${sql(expr)})`, "avg"), nullableNumber) -export const min = (expr: Expr): Expr => { +export const min = (expr: Expr): Expr => { const schema = schemaOf(expr) return makeExpr(aggregate.lazy(() => `min(${sql(expr)})`, "min"), schema === undefined ? undefined : Schema.NullOr(schema)) } -export const max = (expr: Expr): Expr => { +export const max = (expr: Expr): Expr => { const schema = schemaOf(expr) return makeExpr(aggregate.lazy(() => `max(${sql(expr)})`, "max"), schema === undefined ? undefined : Schema.NullOr(schema)) } -export const lower =

(expr: Expr): Expr => +export const lower =

(expr: Expr): Expr => makeExpr(scalar.lazy(() => `lower(${sql(expr)})`, "lower"), Schema.NullOr(Schema.String)) -export const upper =

(expr: Expr): Expr => +export const upper =

(expr: Expr): Expr => makeExpr(scalar.lazy(() => `upper(${sql(expr)})`, "upper"), Schema.NullOr(Schema.String)) /** Text length in SQLite characters, or blob length in bytes. */ -export const length =

(expr: Expr): Expr => +export const length =

(expr: Expr): Expr => makeExpr(scalar.lazy(() => `length(${sql(expr)})`, "length"), Schema.NullOr(Schema.Int)) -export const coalesce = (first: Expr, fallback: Expr): Expr => +export const coalesce = (first: Expr, fallback: Expr): Expr => makeExpr(scalar.lazy(() => `coalesce(${sql(first)}, ${sql(fallback)})`, "coalesce"), schemaOf(fallback)) diff --git a/src/sqlite/sqlite.test.ts b/src/sqlite/sqlite.test.ts index fe5ff92..1f6c27d 100644 --- a/src/sqlite/sqlite.test.ts +++ b/src/sqlite/sqlite.test.ts @@ -45,6 +45,8 @@ it("refuses unsupported SQLite syntax while compiling, before sending SQL", () = .onConflictDoNothing({ target: { constraint: "items_pkey" } }))).toThrow(/ON CONSTRAINT/) expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").distinctOn("id"))).toThrow(/DISTINCT ON/) expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").where(($) => [$.note.ilike("a%")]))).toThrow(/ILIKE/) + // The type refuses it too (see `RowLock`); this is the run-time net. + // @ts-expect-error -- FOR UPDATE has no meaning for the sqlite dialect expect(() => SQLite.compileUnsafe(SQLite.from(Items).select("id").forUpdate())).toThrow(/row locks/) }) diff --git a/tests/fuzz/registry.fuzz.test.ts b/tests/fuzz/registry.fuzz.test.ts index e605d83..86b4dba 100644 --- a/tests/fuzz/registry.fuzz.test.ts +++ b/tests/fuzz/registry.fuzz.test.ts @@ -22,7 +22,8 @@ const sourceFiles = (dir: string): Array => }) const defined = sourceFiles(join(import.meta.dirname, "../../src")).flatMap((file) => - [...readFileSync(file, "utf8").matchAll(/export const \w+: Dialect = \{[\s\S]*?\bname: "([^"]+)"/g)].map((m) => m[1]!), + // `Dialect<"postgres">`: a built-in dialect names its function set in the type. + [...readFileSync(file, "utf8").matchAll(/export const \w+: Dialect(?:<[^>]*>)? = \{[\s\S]*?\bname: "([^"]+)"/g)].map((m) => m[1]!), ) it("every dialect in src/ is registered with the fuzzer", () => {