From ac6aa1de6336257bb74507905ff79f64c8333ce2 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:11:59 +0200 Subject: [PATCH 1/5] feat(query): narrow joined columns proven not NULL by where --- src/ch/compile.ts | 73 ++++++++++++++---- src/ch/expr.ts | 98 ++++++++++++++++++++---- src/ch/facts.ts | 31 ++++++++ src/ch/predicate-facts.test-d.ts | 127 +++++++++++++++++++++++++++++++ src/ch/predicate-facts.test.ts | 91 ++++++++++++++++++++++ src/ch/query.ts | 58 +++++++++++--- 6 files changed, 440 insertions(+), 38 deletions(-) create mode 100644 src/ch/facts.ts create mode 100644 src/ch/predicate-facts.test-d.ts create mode 100644 src/ch/predicate-facts.test.ts diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..c97d286 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -22,7 +22,8 @@ import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder" import { compileQuery, type SqlQuery } from "../sql/sql-query" import { track, untracked, type RenderTrack } from "../sql/render-tracker" import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramColumnTypeOf, paramSchema, type ParamKind } from "./param" -import { mergeResultSchemas } from "./define-fn" +import { mergeResultSchemas, withoutNull } from "./define-fn" +import { notNullFactsOf } from "./facts" import { encodeValue } from "./literal" import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect" import { Effect, Option, Schema } from "effect" @@ -1176,7 +1177,11 @@ function deriveTenantScope( * Shared with `selectExprsOf` below, which needs the same accessor to read a * query's output schemas without compiling it. */ -function makeAccessor(state: CHQueryState): any { +function makeAccessor( + state: CHQueryState, + /** Joined columns proven not NULL, as `alias` → column keys. */ + notNull?: ReadonlyMap>, +): any { const joinAliases = state.typedJoins.map((j) => j.alias) const hasJoins = joinAliases.length > 0 if (!hasJoins) return createQualifiedColumnAccessor(sourceAlias(state), state.tenantColumn, columnsOf(state)) @@ -1188,10 +1193,40 @@ function makeAccessor(state: CHQueryState): any { mainAlias, state.tenantColumn, Object.fromEntries(state.typedJoins.map((j) => [j.alias, j.tenantColumn])), - Object.fromEntries(state.typedJoins.map((j) => [j.alias, joinColumnsOf(j)])), + Object.fromEntries(state.typedJoins.map((j) => [j.alias, joinColumnsOf(j, notNull?.get(j.alias))])), ) } +/** + * The joined columns the WHERE conditions before `select` prove not NULL, + * `undefined` when none. Read off the same top-level AND list whose type + * `where` narrows, so the row schema drops `null` exactly where the type does. + */ +function selectNotNullOf(state: CHQueryState): ReadonlyMap> | undefined { + if (state.selectWhereFn === undefined || state.typedJoins.length === 0) return undefined + const proven = new Map>() + for (const condition of state.selectWhereFn(makeAccessor(state))) { + if (condition == null) continue + for (const fact of notNullFactsOf(condition)) { + const columns = proven.get(fact.source) ?? new Set() + columns.add(fact.column) + proven.set(fact.source, columns) + } + } + return proven.size === 0 ? undefined : proven +} + +/** A column type proven not NULL: a `Nullable` column's inner type, or its schema without the `null` member. */ +function nonNullColumn(type: CHType): CHType { + if (type._tag === "Nullable" && type.element !== undefined) { + return type.sqlName === undefined ? type.element : { ...type.element, sqlName: type.sqlName } + } + const schema = withoutNull(type.schema) + return schema === undefined + ? type + : { ...type, schema, literalSchema: withoutNull(type.literalSchema) ?? type.literalSchema } +} + /** * A join's columns: its table's, or — for a joined subquery — the schemas of * the SELECT it will compile to. @@ -1199,22 +1234,30 @@ function makeAccessor(state: CHQueryState): any { * `innerJoinQuery` records no columns (there is no table to read them from), so * without this every `$.alias.field` off a joined subquery is untyped. */ -function joinColumnsOf(join: { - readonly type: "INNER" | "LEFT" | "CROSS" - readonly columns?: ColumnDefs - readonly innerQuery?: CHQuery -}): ColumnDefs | undefined { +function joinColumnsOf( + join: { + readonly type: "INNER" | "LEFT" | "CROSS" + readonly columns?: ColumnDefs + readonly innerQuery?: CHQuery + }, + /** The columns a WHERE condition proves not NULL: they lose `null`, the LEFT JOIN's included. */ + notNull?: ReadonlySet, +): ColumnDefs | undefined { const exprs = join.innerQuery === undefined ? undefined : selectExprsOf(join.innerQuery) const columns = join.columns ?? (exprs === undefined ? undefined : synthesizeColumns(exprs)) - if (columns === undefined || join.type !== "LEFT") return columns + if (columns === undefined || (join.type !== "LEFT" && notNull === undefined)) return columns return Object.fromEntries( Object.entries(columns).map(([name, type]) => [ name, - { - ...type, - schema: Schema.NullOr(type.schema), - literalSchema: Schema.NullOr(type.literalSchema), - }, + notNull?.has(name) === true + ? nonNullColumn(type) + : join.type !== "LEFT" + ? type + : { + ...type, + schema: Schema.NullOr(type.schema), + literalSchema: Schema.NullOr(type.literalSchema), + }, ]), ) } @@ -1294,6 +1337,8 @@ function selectExprsOf(query: CHQuery): Record | * bare `select()` over a subquery or union, every alias that source selects. */ function selectRecordOf(state: CHQueryState, $: any): Record | undefined { + const notNull = selectNotNullOf(state) + if (notNull !== undefined) $ = makeAccessor(state, notNull) if (state.selectAll) { const source = state.fromUnion ? unionExprsOf(state.fromUnion._state.queries) : state.fromQuery ? selectExprsOf(state.fromQuery) : undefined return source === undefined ? undefined : Object.fromEntries(Object.keys(source).map((alias) => [alias, $[alias]])) diff --git a/src/ch/expr.ts b/src/ch/expr.ts index 99773df..66142d7 100644 --- a/src/ch/expr.ts +++ b/src/ch/expr.ts @@ -16,6 +16,7 @@ import { encodeColumnLiteral, isSubnormal, numberText, sqlLiteral } from "./lite import { QueryBuilderError } from "./errors" import { fail } from "./failure" import { markTenantColumn, markTenantPredicate, tenantColumnOf, tenantPredicatesOf } from "./tenant" +import { markNotNullFacts, notNullFactsOf, type NotNullFact } from "./facts" // Core interfaces @@ -252,10 +253,26 @@ export type Quotient = [R] extends [number] : number | Extract : number | null -export interface ColumnRef> extends Expr< +/** + * A column of a joined source, `Source` its alias. `isNotNull()` and `eq(…)` + * on it prove the column not NULL: the condition carries a `NotNullFact`, and a + * WHERE list that holds it narrows the column for a later `select`. The FROM + * source's columns have no `Source` and carry no fact. + */ +export interface ColumnRef, Source extends string = never> extends Expr< InferTS > { readonly columnName: Name + /** + * phantom: the column's type. Keeps a `ColumnRef` of one column type from + * passing for another when the compiler compares refs member by member, as + * it does for a `Source` it cannot measure (`get` needs a Map column). + */ + readonly _columnType?: ColType + /** `column = other`. True only for a non-NULL column, so it proves one. */ + eq(other: Operand, Q>): Condition> + /** `column IS NOT NULL`. */ + isNotNull(): Condition> /** * Access a key in a Map column: `$.Attrs.get("http.method")`. * @@ -276,15 +293,51 @@ export interface ColumnRef = [ColType] extends [CHType<"Map", Record, any>] ? V : string -export interface Condition

{ +export interface Condition

{ readonly _brand: "Condition" /** phantom: the `param.*` placeholders inside this condition. */ readonly _params?: (entries: P) => void + /** + * phantom: the joined columns this condition proves not NULL, as + * `{ alias: { column: true } }` (see `NotNullFacts`). An object, so a + * condition can drop a fact on assignment but never gain one. + */ + readonly _facts?: F toFragment(): SqlFragment - and(other: Condition): Condition

+ /** Both conditions hold, so the facts of both do. */ + and(other: Condition): Condition

+ /** Either condition may hold: no fact survives. */ or(other: Condition): Condition

} +/** + * The columns proven not NULL: `{ alias: { column: true } }`. `{}` when the + * source is `never`, which is how a FROM column proves nothing. + */ +export type NotNullFacts = { + readonly [S in Source]: { readonly [C in Column]: true } +} + +/** The facts a condition proves, `{}` for none. */ +export type FactsIn = 0 extends 1 & X + ? {} + : X extends { readonly _facts?: infer F } + ? 0 extends 1 & F + ? {} + : Exclude + : {} + +/** + * The facts every condition of a list proves, when all of them are present. + * A list of unknown length may be empty, and a member typed `| undefined` may + * be skipped, so neither proves anything. + */ +export type AllFacts> = number extends C["length"] + ? {} + : FactsIntersection<{ readonly [K in keyof C]: undefined extends C[K] ? {} : FactsIn }[number]> + +type FactsIntersection = (U extends unknown ? (u: U) => void : never) extends (i: infer I) => void ? I & {} : {} + // Core helpers (exported for define-fn.ts and consumer extensibility) /** An already-built expression or condition, rather than a value to encode. */ @@ -610,6 +663,11 @@ export function makeColumnRef { // `alias.Column` when qualified: the qualifier is quoted segment by segment, // the column as one identifier (a ClickHouse `Nested` column has a dot). @@ -645,7 +703,7 @@ export function makeColumnRef encodeColumnLiteral(columnType, value, columnName ?? name)) } - return Object.assign( + const ref = Object.assign( base, isTenantColumn ? { @@ -679,18 +737,29 @@ export function makeColumnRef + if (factSource === undefined) return ref + // `x = y` and `x IS NOT NULL` are never true for a NULL `x`. + const fact: ReadonlyArray = [{ source: factSource, column: columnName ?? name }] + const { eq, isNotNull } = ref + return Object.assign(ref, { + eq: (other: any) => markNotNullFacts(eq(other), fact), + isNotNull: () => markNotNullFacts(isNotNull(), fact), + }) } // Condition implementation -export function makeCond(fragment: SqlFragment): Condition { +export function makeCond(fragment: SqlFragment): Condition { return { _brand: "Condition" as const, toFragment: () => fragment, and(other) { - return markTenantPredicate( - makeCond(known(() => `(${compile(fragment)} AND ${compile(other.toFragment())})`)), - [...tenantPredicatesOf(this), ...tenantPredicatesOf(other)], + return markNotNullFacts( + markTenantPredicate( + makeCond(known(() => `(${compile(fragment)} AND ${compile(other.toFragment())})`)), + [...tenantPredicatesOf(this), ...tenantPredicatesOf(other)], + ), + [...notNullFactsOf(this), ...notNullFactsOf(other)], ) }, or: (other) => makeCond(known(() => `(${compile(fragment)} OR ${compile(other.toFragment())})`)), @@ -749,18 +818,21 @@ export function notInList(expr: Expr, values: readonly string[]): Condit /** * Conditions AND-joined, an `undefined` one skipped: `and(a, when(x, f), b)`. * With none left it is `undefined`, which a `where` list skips in turn. Tenant - * evidence carries through, as with `.and`. + * evidence and not-NULL facts carry through, as with `.and`. */ -export function and>(...conditions: C): Condition> +export function and>(...conditions: C): Condition, AllFacts> export function and>( ...conditions: C ): Condition> | undefined export function and(...conditions: ReadonlyArray): Condition | undefined { const present = conditions.filter((c): c is Condition => c !== undefined) if (present.length <= 1) return present[0] - return markTenantPredicate( - makeCond(known(() => `(${present.map((c) => compile(c.toFragment())).join(" AND ")})`)), - present.flatMap((c) => tenantPredicatesOf(c)), + return markNotNullFacts( + markTenantPredicate( + makeCond(known(() => `(${present.map((c) => compile(c.toFragment())).join(" AND ")})`)), + present.flatMap((c) => tenantPredicatesOf(c)), + ), + present.flatMap((c) => notNullFactsOf(c)), ) } diff --git a/src/ch/facts.ts b/src/ch/facts.ts new file mode 100644 index 0000000..ee39a4a --- /dev/null +++ b/src/ch/facts.ts @@ -0,0 +1,31 @@ +// Predicate facts +// +// A condition in a query's top-level WHERE list is true for every row the +// query returns. Some conditions prove more than their own truth: `x IS NOT +// NULL` and `x = y` are only true when `x` is not NULL. A *fact* records that +// proof for one column of a joined source, so the builder can type and decode +// that column without `| null` once the WHERE list holds it. +// +// The type side rides on `Condition`'s second type parameter; this module is +// the runtime side, which `compile` reads to narrow the row schema the same way. +// Only an AND keeps a fact: `.and`, `and(…)` and the WHERE list itself. An OR, +// a NOT, or an optional `when` branch proves nothing about every row. + +import type { Condition } from "./expr" + +/** A joined column proven not NULL: the join alias and the column key. */ +export interface NotNullFact { + readonly source: Source + readonly column: Column +} + +const facts = new WeakMap>() + +/** The not-NULL facts a condition proves; none for one that proves nothing. */ +export const notNullFactsOf = (condition: Condition): ReadonlyArray => facts.get(condition) ?? [] + +/** Record what `condition` proves. Returns it, for use in an expression. */ +export const markNotNullFacts = (condition: A, proven: ReadonlyArray): A => { + if (proven.length > 0) facts.set(condition, proven) + return condition +} diff --git a/src/ch/predicate-facts.test-d.ts b/src/ch/predicate-facts.test-d.ts new file mode 100644 index 0000000..24ae457 --- /dev/null +++ b/src/ch/predicate-facts.test-d.ts @@ -0,0 +1,127 @@ +// Type-level tests: predicate facts narrow joined columns in the row type + +import { expectTypeOf } from "expect-type" +import * as CH from "./index" +import type { InferQueryOutput } from "./query" + +const Units = CH.table("units", { Id: CH.string, OrgId: CH.string, Name: CH.string }) +const Locks = CH.table("locks", { + Id: CH.string, + UnitId: CH.string, + Battery: CH.uint8, + Serial: CH.nullable(CH.string), +}) + +const joined = CH.from(Units).leftJoin(Locks, "l", (u, l) => u.Id.eq(l.UnitId)) + +type Row = InferQueryOutput +type StillNullable = { readonly lockId: string | null } + +// A LEFT JOIN makes the joined columns nullable + +const plain = joined.select(($) => ({ lockId: $.l.Id, battery: $.l.Battery })) +expectTypeOf>().toEqualTypeOf<{ + readonly lockId: string | null + readonly battery: number | null +}>() + +// isNotNull() on a joined column narrows it for a later select + +const notNull = joined + .where(($) => [$.OrgId.eq("o1"), $.l.Id.isNotNull()]) + .select(($) => ({ lockId: $.l.Id, battery: $.l.Battery })) +expectTypeOf>().toEqualTypeOf<{ + readonly lockId: string + readonly battery: number | null +}>() + +// eq() proves its column not NULL too: `NULL = x` is never true + +const byEq = joined + .where(($) => [$.l.Battery.eq(CH.param.int("battery"))]) + .select(($) => ({ lockId: $.l.Id, battery: $.l.Battery })) +expectTypeOf>().toEqualTypeOf<{ + readonly lockId: string | null + readonly battery: number +}>() + +// A column declared Nullable loses both NULLs + +const serial = joined.where(($) => [$.l.Serial.isNotNull()]).select(($) => ({ serial: $.l.Serial })) +expectTypeOf>().toEqualTypeOf<{ readonly serial: string }>() + +// AND chains keep every fact: `.and`, `CH.and`, and a second `where` + +const chained = joined + .where(($) => [$.l.Id.isNotNull().and($.l.Battery.eq(1))]) + .select(($) => ({ lockId: $.l.Id, battery: $.l.Battery })) +expectTypeOf>().toEqualTypeOf<{ readonly lockId: string; readonly battery: number }>() + +const andFn = joined + .where(($) => [CH.and($.l.Id.isNotNull(), $.l.Battery.eq(1))]) + .select(($) => ({ lockId: $.l.Id, battery: $.l.Battery })) +expectTypeOf>().toEqualTypeOf<{ readonly lockId: string; readonly battery: number }>() + +const twoWheres = joined + .where(($) => [$.l.Id.isNotNull()]) + .where(($) => [$.l.Battery.eq(1)]) + .select(($) => ({ lockId: $.l.Id, battery: $.l.Battery })) +expectTypeOf>().toEqualTypeOf<{ readonly lockId: string; readonly battery: number }>() + +// What proves nothing about every row keeps `| null` + +const viaOr = joined + .where(($) => [CH.or($.l.Id.isNotNull(), $.Name.eq("a"))]) + .select(($) => ({ lockId: $.l.Id })) +expectTypeOf>().toEqualTypeOf() + +const viaOrMethod = joined + .where(($) => [$.l.Id.isNotNull().or($.Name.eq("a"))]) + .select(($) => ({ lockId: $.l.Id })) +expectTypeOf>().toEqualTypeOf() + +const viaNot = joined.where(($) => [CH.not($.l.Id.isNull())]).select(($) => ({ lockId: $.l.Id })) +expectTypeOf>().toEqualTypeOf() + +declare const flag: boolean +const viaWhen = joined + .where(($) => [CH.whenTrue(flag, () => $.l.Id.isNotNull())]) + .select(($) => ({ lockId: $.l.Id })) +expectTypeOf>().toEqualTypeOf() + +const viaMaybe = joined + .where(($) => [flag ? $.l.Id.isNotNull() : undefined]) + .select(($) => ({ lockId: $.l.Id })) +expectTypeOf>().toEqualTypeOf() + +declare const conditions: ReadonlyArray +const viaArray = joined.where(() => conditions).select(($) => ({ lockId: $.l.Id })) +expectTypeOf>().toEqualTypeOf() + +// The fact names its column: another column of the same join stays nullable + +const otherColumn = joined.where(($) => [$.l.Battery.isNotNull()]).select(($) => ({ lockId: $.l.Id })) +expectTypeOf>().toEqualTypeOf() + +// A select made before the where keeps `| null`: it cannot see the later fact + +const selectFirst = joined.select(($) => ({ lockId: $.l.Id })).where(($) => [$.l.Id.isNotNull()]) +expectTypeOf>().toEqualTypeOf() + +// A FROM column proves nothing: the joined columns stay as they were + +const fromOnly = joined.where(($) => [$.Id.isNotNull(), $.Name.eq("a")]) +expectTypeOf(fromOnly).toEqualTypeOf() + +// leftJoinQuery: the subquery's columns narrow the same way + +const perUnit = CH.from(Locks).select(($) => ({ unitId: $.UnitId, serial: $.Serial })) +const sub = CH.from(Units) + .leftJoinQuery(perUnit, "p", (u, p) => u.Id.eq(p.unitId)) + .where(($) => [$.p.unitId.isNotNull(), $.p.serial.isNotNull()]) + .select(($) => ({ unitId: $.p.unitId, serial: $.p.serial })) +expectTypeOf>().toEqualTypeOf<{ readonly unitId: string; readonly serial: string }>() + +// The compiled query carries the narrowed row + +expectTypeOf(CH.compileUnsafe(notNull, {})).toEqualTypeOf, undefined>>() diff --git a/src/ch/predicate-facts.test.ts b/src/ch/predicate-facts.test.ts new file mode 100644 index 0000000..25d1df8 --- /dev/null +++ b/src/ch/predicate-facts.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "@effect/vitest" +import { Effect, Exit } from "effect" +import * as CH from "./index" +import * as PG from "../postgres" + +const Units = CH.table("units", { Id: CH.string, Name: CH.string }) +const Locks = CH.table("locks", { Id: CH.string, UnitId: CH.string, Battery: CH.uint8, Serial: CH.nullable(CH.string) }) +const joined = CH.from(Units).leftJoin(Locks, "l", (u, l) => u.Id.eq(l.UnitId)) + +const decodes = (compiled: { decodeRows: (rows: ReadonlyArray>) => Effect.Effect }, row: Record) => + Exit.isSuccess(Effect.runSyncExit(compiled.decodeRows([row]))) + +describe("predicate facts narrow the row schema", () => { + it("decodes a LEFT JOIN column as nullable without a fact", () => { + const compiled = CH.compileUnsafe(joined.select(($) => ({ lockId: $.l.Id })), {}) + expect(decodes(compiled, { lockId: null })).toBe(true) + }) + + it("refuses NULL for a column isNotNull() proves present", () => { + const compiled = CH.compileUnsafe( + joined.where(($) => [$.l.Id.isNotNull()]).select(($) => ({ lockId: $.l.Id, battery: $.l.Battery })), + {}, + ) + expect(decodes(compiled, { lockId: "k1", battery: null })).toBe(true) + expect(decodes(compiled, { lockId: null, battery: null })).toBe(false) + expect(compiled.sql).toContain("l.Id IS NOT NULL") + }) + + it("narrows on eq() and on AND chains", () => { + const compiled = CH.compileUnsafe( + joined + .where(($) => [CH.and($.l.Battery.eq(CH.param.int("b")), $.l.Id.isNotNull().and($.l.UnitId.eq($.Id)))]) + .select(($) => ({ lockId: $.l.Id, battery: $.l.Battery, unitId: $.l.UnitId })), + { b: 3 }, + ) + expect(decodes(compiled, { lockId: "k1", battery: 3, unitId: "u1" })).toBe(true) + for (const key of ["lockId", "battery", "unitId"]) { + expect(decodes(compiled, { lockId: "k1", battery: 3, unitId: "u1", [key]: null })).toBe(false) + } + }) + + it("strips the column's own NULL as well as the join's", () => { + const compiled = CH.compileUnsafe(joined.where(($) => [$.l.Serial.isNotNull()]).select(($) => ({ serial: $.l.Serial })), {}) + expect(decodes(compiled, { serial: "s" })).toBe(true) + expect(decodes(compiled, { serial: null })).toBe(false) + }) + + it("keeps NULL where the type keeps it: OR, NOT, and a select before the where", () => { + const or = CH.compileUnsafe( + joined.where(($) => [CH.or($.l.Id.isNotNull(), $.Name.eq("a"))]).select(($) => ({ lockId: $.l.Id })), + {}, + ) + const orMethod = CH.compileUnsafe( + joined.where(($) => [$.l.Id.isNotNull().or($.Name.eq("a"))]).select(($) => ({ lockId: $.l.Id })), + {}, + ) + const not = CH.compileUnsafe(joined.where(($) => [CH.not($.l.Id.isNull())]).select(($) => ({ lockId: $.l.Id })), {}) + const selectFirst = CH.compileUnsafe( + joined.select(($) => ({ lockId: $.l.Id })).where(($) => [$.l.Id.isNotNull()]), + {}, + ) + for (const compiled of [or, orMethod, not, selectFirst]) expect(decodes(compiled, { lockId: null })).toBe(true) + expect(selectFirst.sql).toContain("l.Id IS NOT NULL") + }) + + it("narrows a leftJoinQuery column, and a FROM subquery over the narrowed query", () => { + const perUnit = CH.from(Locks).select(($) => ({ unitId: $.UnitId, serial: $.Serial })) + const narrowed = CH.from(Units) + .leftJoinQuery(perUnit, "p", (u, p) => u.Id.eq(p.unitId)) + .where(($) => [$.p.serial.isNotNull()]) + .select(($) => ({ unitId: $.p.unitId, serial: $.p.serial })) + expect(decodes(CH.compileUnsafe(narrowed, {}), { unitId: null, serial: "s" })).toBe(true) + expect(decodes(CH.compileUnsafe(narrowed, {}), { unitId: null, serial: null })).toBe(false) + + const outer = CH.fromQuery(narrowed, "n").select(($) => ({ serial: $.serial })) + expect(decodes(CH.compileUnsafe(outer, {}), { serial: null })).toBe(false) + }) + + it("narrows the same way on Postgres", () => { + const Members = CH.table("members", { id: PG.int4, orgId: PG.text, leftAt: PG.nullable(PG.timestamptz) }) + const compiled = PG.compileUnsafe( + CH.from(Units) + .leftJoin(Members, "m", (u, m) => u.Id.eq(m.orgId)) + .where(($) => [$.m.leftAt.isNotNull()]) + .select(($) => ({ leftAt: $.m.leftAt, id: $.m.id })), + {}, + ) + expect(decodes(compiled, { leftAt: "2026-01-01T00:00:00Z", id: null })).toBe(true) + expect(decodes(compiled, { leftAt: null, id: 1 })).toBe(false) + }) +}) diff --git a/src/ch/query.ts b/src/ch/query.ts index 36752b4..7ba0664 100644 --- a/src/ch/query.ts +++ b/src/ch/query.ts @@ -26,7 +26,7 @@ import type { ColumnDefs, CHType, InferTS, OutputToColumnDefs, NullableColumnDefs } from "./types" import type { Table } from "./table" -import type { Expr, Condition, ColumnRef, ParamsIn, Widen } from "./expr" +import type { AllFacts, Expr, Condition, ColumnRef, ParamsIn, Widen } from "./expr" import { makeColumnRef } from "./expr" import type { TenantScope } from "./compile" @@ -51,8 +51,9 @@ const selectEvery = ($: any): Record => Object.fromEntries(columns.map((column) => [column, $[column]])) -export type ColumnAccessor = { - readonly [K in keyof Cols & string]: ColumnRef +/** A source's columns. `Source`, a join alias, makes them record `NotNullFact`s. */ +export type ColumnAccessor = { + readonly [K in keyof Cols & string]: ColumnRef } /** Combined accessor: main table columns + nested alias accessors for joins. */ @@ -60,9 +61,29 @@ export type JoinedColumnAccessor< Cols extends ColumnDefs, Joins extends Record, > = ColumnAccessor & { - readonly [A in keyof Joins & string]: ColumnAccessor + readonly [A in keyof Joins & string]: ColumnAccessor } +/** A column type without its NULL. */ +type NonNullColumn = C extends CHType ? CHType, NonNullable> : C + +/** + * The joined sources once the facts `F` (`{ alias: { column: true } }`) hold: + * each proven column loses its `| null`. Unchanged when nothing is proven, so + * a query without facts keeps the exact `Joins` type it had. + */ +export type NarrowJoins, F> = {} extends F + ? Joins + : { + readonly [A in keyof Joins]: { + readonly [K in keyof Joins[A]]: A extends keyof F + ? K extends keyof F[A] + ? NonNullColumn + : Joins[A][K] + : Joins[A][K] + } + } + type SelectRecord = Record> /** @@ -133,6 +154,12 @@ export interface CHQueryState { */ readonly selectAll?: true readonly whereFn?: ($: any) => ReadonlyArray + /** + * The WHERE conditions there were when `select` was called. Their facts + * narrow the SELECT's joined columns, as `where`'s type narrows them for a + * later `select` only. + */ + readonly selectWhereFn?: ($: any) => ReadonlyArray readonly groupByKeys: string[] /** Post-aggregation filter. Deliberately NOT consulted when deriving tenant * scope — see `having()` on the interface. */ @@ -253,10 +280,16 @@ export interface CHQuery< /** * Filter rows: conditions AND-joined, an `undefined` one skipped. Calling it * again adds conditions, ANDed with the earlier ones. + * + * A condition that proves a joined column not NULL (`$.o.Id.isNotNull()`, + * `$.o.Status.eq(…)`, alone or under `and`) narrows that column for the + * calls after this one: a later `select` reads it without `| null`, in the + * type and in the row schema. A `select` made before the `where` keeps + * `| null`. */ where>( fn: ($: JoinedColumnAccessor) => C, - ): CHQuery> + ): CHQuery>, Route, Params | ParamsIn> groupBy(...keys: Array): CHQuery @@ -487,12 +520,14 @@ export function createQualifiedColumnAccessor( alias: string, tenantColumn?: string, columns?: ColumnDefs, + /** Set for a joined source: its columns record `NotNullFact`s under this alias. */ + factSource?: string, ): ColumnAccessor { const cache = new Map>>() const get = (prop: string) => { let ref = cache.get(prop) if (!ref) { - ref = makeColumnRef(`${alias}.${prop}`, prop, tenantColumn, columns?.[prop]) + ref = makeColumnRef(`${alias}.${prop}`, prop, tenantColumn, columns?.[prop], factSource) cache.set(prop, ref) } return ref @@ -531,7 +566,7 @@ export function createJoinedColumnAccessor ({ Col1: $.Col1, Col2: $.Col2 })) - if (typeof args[0] === "string") return makeQuery({ ...state, selectFn: selectEvery(args as string[]), selectAll: undefined }) + if (typeof args[0] === "string") return makeQuery({ ...selected, selectFn: selectEvery(args as string[]), selectAll: undefined }) // Callback overload: select($ => ({ ... })) - return makeQuery({ ...state, selectFn: args[0], selectAll: undefined }) + return makeQuery({ ...selected, selectFn: args[0], selectAll: undefined }) }, where(fn) { From b60f069c860ce45b2fe35a7ae7f11a6c7fbd4174 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:11:59 +0200 Subject: [PATCH 2/5] test: run the not-NULL narrowing live and in consumer declarations --- tests/__snapshots__/core-sql.test.ts.snap | 45 +++++++++++++++++++++++ tests/core-cases.ts | 23 ++++++++++++ tests/package-consumer-declarations.mts | 9 +++++ 3 files changed, 77 insertions(+) diff --git a/tests/__snapshots__/core-sql.test.ts.snap b/tests/__snapshots__/core-sql.test.ts.snap index fba9e28..612c40c 100644 --- a/tests/__snapshots__/core-sql.test.ts.snap +++ b/tests/__snapshots__/core-sql.test.ts.snap @@ -340,6 +340,27 @@ SELECT } `; +exports[`core SQL (clickhouse) > left-join-not-null 1`] = ` +{ + "parameters": [], + "sql": "WITH orders AS ( + +), +customers AS ( + +) +SELECT + orders.Id AS id, + c.Tier AS tier + FROM orders + LEFT JOIN customers AS c ON (orders.Customer = c.Name AND orders.OrgId = c.OrgId) + WHERE orders.OrgId = 'org_1' + AND orders.Id >= 3 + AND c.Tier IS NOT NULL + ORDER BY id ASC", +} +`; + exports[`core SQL (clickhouse) > limit-offset 1`] = ` { "parameters": [], @@ -970,6 +991,30 @@ SELECT } `; +exports[`core SQL (postgres) > left-join-not-null 1`] = ` +{ + "parameters": [ + "org_1", + 3, + ], + "sql": "WITH "orders" AS ( + +), +"customers" AS ( + +) +SELECT + "orders"."Id" AS "id", + "c"."Tier" AS "tier" + FROM "orders" + LEFT JOIN "customers" AS "c" ON ("orders"."Customer" = "c"."Name" AND "orders"."OrgId" = "c"."OrgId") + WHERE "orders"."OrgId" = $1 + AND "orders"."Id" >= $2 + AND "c"."Tier" IS NOT NULL + ORDER BY "id" ASC", +} +`; + exports[`core SQL (postgres) > limit-offset 1`] = ` { "parameters": [ diff --git a/tests/core-cases.ts b/tests/core-cases.ts index 3698368..4cec4ad 100644 --- a/tests/core-cases.ts +++ b/tests/core-cases.ts @@ -421,6 +421,29 @@ export const coreCases: readonly CoreCase[] = [ ], }, }, + { + // A where before the select narrows the joined column, so the row + // schema refuses NULL for it. Without join_use_nulls the missing row + // has the column default, which is not NULL and passes the filter. + id: "left-join-not-null", + covers: q("leftJoin", "where"), + build: (ctx) => + ctx.compile( + orgOrders(ctx) + .leftJoin(ctx.customers, "c", (o, c) => o.Customer.eq(c.Name).and(o.OrgId.eq(c.OrgId))) + .where(($) => [$.Id.gte(3), $.c.Tier.isNotNull()]) + .select(($) => ({ id: $.Id, tier: $.c.Tier })) + .orderBy(["id", "asc"]), + org, + ), + expected: [{ id: 3, tier: "silver" }], + expectedBy: { + clickhouse: [ + { id: 3, tier: "silver" }, + { id: 4, tier: "" }, + ], + }, + }, { id: "cross-join", covers: q("crossJoin"), diff --git a/tests/package-consumer-declarations.mts b/tests/package-consumer-declarations.mts index 616c47b..6573222 100644 --- a/tests/package-consumer-declarations.mts +++ b/tests/package-consumer-declarations.mts @@ -47,3 +47,12 @@ export const Recent = CH.table("recent", { external: true, columns: { name: CH.s export const withRecent = CH.from(Recent).withCTE("recent", eventsByApp).select(($) => ({ name: $.name })) export const fromSub = CH.fromQuery(eventsByApp, "sub").select(($) => ({ name: $.name })) export const both = CH.unionAll(eventsByApp, CH.from(Events).select(($) => ({ name: $.name }))) + +// A where that proves a joined column not NULL: the query's type carries +// `NarrowJoins`, and a fact-carrying condition `NotNullFacts`. +export const sessionNotNull = CH.from(Events) + .leftJoin(Sessions, "s", (e, s) => e.appId.eq(s.appId)) + .where(($) => [$.s.id.isNotNull()]) + .select(($) => ({ sessionId: $.s.id })) +export const sessionFilter = (accessor: CH.JoinedColumnAccessor) => + accessor.s.id.isNotNull() From 46a5b107f790d0ad7ad4139ac124e34d1d83bf8a Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:11:59 +0200 Subject: [PATCH 3/5] docs: document where narrowing of joined columns --- docs/joins-and-subqueries.md | 33 +++++++++++++++++++++++++++++++++ docs/reference.md | 8 ++++++-- src/docs-examples.test.ts | 19 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/docs/joins-and-subqueries.md b/docs/joins-and-subqueries.md index 788e67b..b4d33ca 100644 --- a/docs/joins-and-subqueries.md +++ b/docs/joins-and-subqueries.md @@ -34,6 +34,39 @@ in the output row. Derived codecs accept NULL when `join_use_nulls=1` and the co defaults ClickHouse emits with `join_use_nulls=0`. This applies to `leftJoinQuery` too. An explicitly declared `rowSchema` must also accept the nullable joined columns. +### Narrowing a joined column with `where` + +A `where` condition can prove that a joined column is not NULL. Then a `select` after that +`where` reads the column without `| null`, in the row type and in the derived row schema: + +```ts +const query = CH.from(Events, "e") + .leftJoin(Services, "s", (main, joined) => main.Name.eq(joined.Name).and(main.OrgId.eq(joined.OrgId))) + .where(($) => [$.OrgId.eq("org_123"), $.s.Team.isNotNull()]) + .select(($) => ({ name: $.Name, team: $.s.Team })) + +// Row: { readonly name: string; readonly team: string } +``` + +These conditions prove a column not NULL: + +- `$.s.Team.isNotNull()`. +- `$.s.Team.eq(value)`: `NULL = value` is never true. +- An AND of them: `.and(…)`, `CH.and(…)`, the items of a `where` list, or a second `where`. + +These conditions prove nothing, so the column stays `T | null`: + +- `or`, `.or`, `not`, `when` and `whenTrue`. +- An item typed `Condition | undefined`, or a list of unknown length. +- A condition on a column of the FROM source. + +Call `where` before `select`. A `select` made before the `where` keeps `| null`, because its +row type is fixed when you call it. The row schema follows the same rule, so the type and the +schema agree. On ClickHouse without `join_use_nulls=1`, a missing join row has the column +default, which is not NULL: `isNotNull()` keeps that row. + +_(Backed by `docs/joins-and-subqueries.md > A where before select narrows a joined column`.)_ + ## Joining a subquery `innerJoinQuery`, `leftJoinQuery`, and `crossJoinQuery` take a `CHQuery` instead of a table. diff --git a/docs/reference.md b/docs/reference.md index abc2d46..f2af096 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -56,7 +56,7 @@ Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or | Method | Notes | | ------------------------------------------------------- | --------------------------------------------------------------------- | | `select(...names)` / `select(fn)` | Required before compiling | -| `where(fn)` | Returns `Array`; AND-joined | +| `where(fn)` | Returns `Array`; AND-joined. Narrows a joined column it proves not NULL for a later `select` ([joins](./joins-and-subqueries.md#narrowing-a-joined-column-with-where)) | | `groupBy(...outputKeys)` | Takes select aliases, not column names | | `having(fn)` | Post-aggregation filter; input accessor or `dynamicColumn` aliases | | `orderBy(...[col, dir])` | **Tuples**, not two strings | @@ -144,7 +144,11 @@ outer set and its failures land in the outer error channel. See `wrap` receives the inner SQL and returns the text to emit. It defaults to wrapping the SQL in parentheses, which is the plain "this value is a sub-SELECT" case. -`Condition` methods: `and`, `or` (both parenthesise; both drop the tenant marker). +`Condition` methods: `and`, `or` (both parenthesise; both drop the tenant marker). `and` +keeps the not-NULL facts of both sides; `or` keeps none. + +On a joined column, `isNotNull()` and `eq(…)` return a `Condition` that proves the column not +NULL. Its second type parameter holds that fact. `ColumnRef` adds `.get(key)` for `Map` columns; the result decodes as the map's value type. diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index ef673be..3b53048 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -14,6 +14,7 @@ import type { DateTime } from "effect" import { describe, expect, it } from "@effect/vitest" +import { expectTypeOf } from "expect-type" import { Effect, Exit, Option, Schema } from "effect" import * as CH from "./ch/index" import * as T from "./ch/types" @@ -397,6 +398,24 @@ describe("docs/joins-and-subqueries.md", () => { expect(compileCHUnsafe(query, {}).tenantScope).toBe("single-tenant") }) + it("A where before select narrows a joined column", () => { + const query = CH.from(Events, "e") + .leftJoin(Services, "s", (main, joined) => main.Name.eq(joined.Name).and(main.OrgId.eq(joined.OrgId))) + .where(($) => [$.OrgId.eq("org_123"), $.s.Team.isNotNull()]) + .select(($) => ({ name: $.Name, team: $.s.Team })) + const compiled = compileCHUnsafe(query, {}) + + expectTypeOf(compiled.decodeRows).returns.toEqualTypeOf< + Effect.Effect, CH.CompiledQueryDecodeError> + >() + expect(oneLine(compiled.sql)).toBe( + "SELECT e.Name AS name, s.Team AS team FROM events AS e " + + "LEFT JOIN services AS s ON (e.Name = s.Name AND e.OrgId = s.OrgId) " + + "WHERE e.OrgId = 'org_123' AND s.Team IS NOT NULL", + ) + expect(Exit.isFailure(Effect.runSyncExit(compiled.decodeRows([{ name: "api", team: null }])))).toBe(true) + }) + it("Subquery in FROM uses flat accessors", () => { const inner = CH.from(Events) .select(($) => ({ name: $.Name, ms: $.DurationMs })) From 26a90c7389167e247e659af646db5520eba3c97a Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:11:59 +0200 Subject: [PATCH 4/5] chore: add changeset for predicate facts --- .changeset/predicate-facts-joins.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/predicate-facts-joins.md diff --git a/.changeset/predicate-facts-joins.md b/.changeset/predicate-facts-joins.md new file mode 100644 index 0000000..abd162c --- /dev/null +++ b/.changeset/predicate-facts-joins.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +A `where` that proves a joined column not NULL now narrows that column for a later `select`. `$.l.Id.isNotNull()` and `$.l.Id.eq(…)` prove it, alone or under `.and(…)`, `and(…)` or the `where` list. The row type and the derived row schema both drop `| null`, so `leftJoin(…).where(($) => [$.l.Id.isNotNull()]).select(($) => ({ id: $.l.Id }))` reads `id: string`, and a NULL in that column fails to decode. `or`, `not`, `when` and a `select` made before the `where` keep `| null`. `Condition` has a second type parameter for these facts; it defaults to none, so existing code does not change. From 04ce9c5c29f01fa381e0d6daad704d33736d807a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 2 Nov 2026 16:44:59 +0100 Subject: [PATCH 5/5] test(query): narrow joined columns on SQLite, D1 and the /postgres entry The narrowing landed before the /sqlite entry existed. Check that its nullable type narrows in the row type and in the row schema (also under d1Dialect), and that the /postgres entry's own types narrow on eq(). Co-Authored-By: Claude Opus 5.5 --- src/ch/predicate-facts.test-d.ts | 20 ++++++++++++++++++++ src/ch/predicate-facts.test.ts | 15 +++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/ch/predicate-facts.test-d.ts b/src/ch/predicate-facts.test-d.ts index 24ae457..0cb9704 100644 --- a/src/ch/predicate-facts.test-d.ts +++ b/src/ch/predicate-facts.test-d.ts @@ -2,6 +2,8 @@ import { expectTypeOf } from "expect-type" import * as CH from "./index" +import * as PG from "../postgres" +import * as SQLite from "../sqlite" import type { InferQueryOutput } from "./query" const Units = CH.table("units", { Id: CH.string, OrgId: CH.string, Name: CH.string }) @@ -125,3 +127,21 @@ expectTypeOf>().toEqualTypeOf<{ readonly unitId: string; readonl // The compiled query carries the narrowed row expectTypeOf(CH.compileUnsafe(notNull, {})).toEqualTypeOf, undefined>>() + +// The /sqlite and /postgres entries narrow the same way, their own nullable types included + +const Accounts = SQLite.table("accounts", { id: SQLite.text }) +const Profiles = SQLite.table("profiles", { accountId: SQLite.text, bio: SQLite.nullable(SQLite.text), age: SQLite.integer }) +const sqliteNarrowed = SQLite.from(Accounts) + .leftJoin(Profiles, "p", (a, p) => a.id.eq(p.accountId)) + .where(($) => [$.p.bio.isNotNull()]) + .select(($) => ({ bio: $.p.bio, age: $.p.age })) +expectTypeOf>().toEqualTypeOf<{ readonly bio: string; readonly age: number | null }>() +expectTypeOf(SQLite.compileUnsafe(sqliteNarrowed, {})).toEqualTypeOf, undefined>>() + +const Members = PG.table("members", { columns: { id: PG.int4, accountId: PG.text, leftAt: PG.nullable(PG.text) } }) +const pgNarrowed = PG.from(Accounts) + .leftJoin(Members, "m", (a, m) => a.id.eq(m.accountId)) + .where(($) => [$.m.leftAt.eq("x")]) + .select(($) => ({ leftAt: $.m.leftAt, id: $.m.id })) +expectTypeOf>().toEqualTypeOf<{ readonly leftAt: string; readonly id: number | null }>() diff --git a/src/ch/predicate-facts.test.ts b/src/ch/predicate-facts.test.ts index 25d1df8..547ed02 100644 --- a/src/ch/predicate-facts.test.ts +++ b/src/ch/predicate-facts.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "@effect/vitest" import { Effect, Exit } from "effect" import * as CH from "./index" import * as PG from "../postgres" +import * as SQLite from "../sqlite" const Units = CH.table("units", { Id: CH.string, Name: CH.string }) const Locks = CH.table("locks", { Id: CH.string, UnitId: CH.string, Battery: CH.uint8, Serial: CH.nullable(CH.string) }) @@ -88,4 +89,18 @@ describe("predicate facts narrow the row schema", () => { expect(decodes(compiled, { leftAt: "2026-01-01T00:00:00Z", id: null })).toBe(true) expect(decodes(compiled, { leftAt: null, id: 1 })).toBe(false) }) + + it("narrows the same way on SQLite and D1", () => { + const Accounts = SQLite.table("accounts", { id: SQLite.text }) + const Profiles = SQLite.table("profiles", { accountId: SQLite.text, bio: SQLite.nullable(SQLite.text), age: SQLite.integer }) + const query = SQLite.from(Accounts) + .leftJoin(Profiles, "p", (a, p) => a.id.eq(p.accountId)) + .where(($) => [$.p.bio.isNotNull()]) + .select(($) => ({ bio: $.p.bio, age: $.p.age })) + for (const compiled of [SQLite.compileUnsafe(query, {}), SQLite.compileUnsafe(query, {}, { dialect: SQLite.d1Dialect })]) { + expect(compiled.sql).toContain(`"p"."bio" IS NOT NULL`) + expect(decodes(compiled, { bio: "hi", age: null })).toBe(true) + expect(decodes(compiled, { bio: null, age: 1 })).toBe(false) + } + }) })