feat(query): narrow joined columns proven not NULL by where

.changeset/predicate-facts-joins.md

@@ -0,0 +1,5 @@
+---
+"@maple-dev/effect-orm": minor
+---
+
+A `where` that proves a joined column not NULL now narrows that column for a later `select`. `$.l.Id.isNotNull()` and `$.l.Id.eq(…)` prove it, alone or under `.and(…)`, `and(…)` or the `where` list. The row type and the derived row schema both drop `| null`, so `leftJoin(…).where(($) => [$.l.Id.isNotNull()]).select(($) => ({ id: $.l.Id }))` reads `id: string`, and a NULL in that column fails to decode. `or`, `not`, `when` and a `select` made before the `where` keep `| null`. `Condition` has a second type parameter for these facts; it defaults to none, so existing code does not change.

docs/joins-and-subqueries.md

@@ -34,6 +34,39 @@ in the output row. Derived codecs accept NULL when `join_use_nulls=1` and the co
 defaults ClickHouse emits with `join_use_nulls=0`. This applies to `leftJoinQuery` too.
 An explicitly declared `rowSchema` must also accept the nullable joined columns.
 
+### Narrowing a joined column with `where`
+
+A `where` condition can prove that a joined column is not NULL. Then a `select` after that
+`where` reads the column without `| null`, in the row type and in the derived row schema:
+
+```ts
+const query = CH.from(Events, "e")
+	.leftJoin(Services, "s", (main, joined) => main.Name.eq(joined.Name).and(main.OrgId.eq(joined.OrgId)))
+	.where(($) => [$.OrgId.eq("org_123"), $.s.Team.isNotNull()])
+	.select(($) => ({ name: $.Name, team: $.s.Team }))
+
+// Row: { readonly name: string; readonly team: string }
+```
+
+These conditions prove a column not NULL:
+
+- `$.s.Team.isNotNull()`.
+- `$.s.Team.eq(value)`: `NULL = value` is never true.
+- An AND of them: `.and(…)`, `CH.and(…)`, the items of a `where` list, or a second `where`.
+
+These conditions prove nothing, so the column stays `T | null`:
+
+- `or`, `.or`, `not`, `when` and `whenTrue`.
+- An item typed `Condition | undefined`, or a list of unknown length.
+- A condition on a column of the FROM source.
+
+Call `where` before `select`. A `select` made before the `where` keeps `| null`, because its
+row type is fixed when you call it. The row schema follows the same rule, so the type and the
+schema agree. On ClickHouse without `join_use_nulls=1`, a missing join row has the column
+default, which is not NULL: `isNotNull()` keeps that row.
+
+_(Backed by `docs/joins-and-subqueries.md > A where before select narrows a joined column`.)_
+
 ## Joining a subquery
 
 `innerJoinQuery`, `leftJoinQuery`, and `crossJoinQuery` take a `CHQuery` instead of a table.

docs/reference.md

@@ -57,7 +57,7 @@ Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or
 | Method                                                  | Notes                                                                 |
 | ------------------------------------------------------- | --------------------------------------------------------------------- |
 | `select(...names)` / `select(fn)`                       | Required before compiling                                             |
-| `where(fn)`                                             | Returns `Array<Condition \| undefined>`; AND-joined                   |
+| `where(fn)`                                             | Returns `Array<Condition \| undefined>`; AND-joined. Narrows a joined column it proves not NULL for a later `select` ([joins](./joins-and-subqueries.md#narrowing-a-joined-column-with-where)) |
 | `groupBy(...outputKeys)`                                | Takes select aliases, not column names                                |
 | `having(fn)`                                            | Post-aggregation filter; input accessor or `dynamicColumn` aliases    |
 | `orderBy(...[col, dir])`                                | **Tuples**, not two strings                                           |
@@ -145,7 +145,11 @@ outer set and its failures land in the outer error channel. See
 `wrap` receives the inner SQL and returns the text to emit. It defaults to wrapping the SQL in
 parentheses, which is the plain "this value is a sub-SELECT" case.
 
-`Condition` methods: `and`, `or` (both parenthesise; both drop the tenant marker).
+`Condition` methods: `and`, `or` (both parenthesise; both drop the tenant marker). `and`
+keeps the not-NULL facts of both sides; `or` keeps none.
+
+On a joined column, `isNotNull()` and `eq(…)` return a `Condition` that proves the column not
+NULL. Its second type parameter holds that fact.
 
 `ColumnRef` adds `.get(key)` for `Map` columns; the result decodes as the map's value type.
 

src/ch/compile.ts

@@ -22,7 +22,8 @@ import { activeLiteralBinder, withLiteralBinder } from "../sql/literal-binder"
 import { compileQuery, type SqlQuery } from "../sql/sql-query"
 import { track, untracked, type RenderTrack } from "../sql/render-tracker"
 import { PARAM_MARKER_PREFIX, PARAM_PLACEHOLDER_PATTERN, param, paramColumnTypeOf, paramSchema, type ParamKind } from "./param"
-import { mergeResultSchemas, schemaOf } from "./define-fn"
+import { mergeResultSchemas, schemaOf, withoutNull } from "./define-fn"
+import { notNullFactsOf } from "./facts"
 import { encodeValue } from "./literal"
 import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Dialect } from "./dialect"
 import { Effect, Option, Schema } from "effect"
@@ -1180,7 +1181,11 @@ function deriveTenantScope(
  * Shared with `selectExprsOf` below, which needs the same accessor to read a
  * query's output schemas without compiling it.
  */
-function makeAccessor(state: CHQueryState): any {
+function makeAccessor(
+	state: CHQueryState,
+	/** Joined columns proven not NULL, as `alias` → column keys. */
+	notNull?: ReadonlyMap<string, ReadonlySet<string>>,
+): any {
 	const joinAliases = state.typedJoins.map((j) => j.alias)
 	const hasJoins = joinAliases.length > 0
 	if (!hasJoins) return createQualifiedColumnAccessor(sourceAlias(state), state.tenantColumn, columnsOf(state))
@@ -1192,33 +1197,71 @@ function makeAccessor(state: CHQueryState): any {
 		mainAlias,
 		state.tenantColumn,
 		Object.fromEntries(state.typedJoins.map((j) => [j.alias, j.tenantColumn])),
-		Object.fromEntries(state.typedJoins.map((j) => [j.alias, joinColumnsOf(j)])),
+		Object.fromEntries(state.typedJoins.map((j) => [j.alias, joinColumnsOf(j, notNull?.get(j.alias))])),
 	)
 }
 
+/**
+ * The joined columns the WHERE conditions before `select` prove not NULL,
+ * `undefined` when none. Read off the same top-level AND list whose type
+ * `where` narrows, so the row schema drops `null` exactly where the type does.
+ */
+function selectNotNullOf(state: CHQueryState): ReadonlyMap<string, ReadonlySet<string>> | undefined {
+	if (state.selectWhereFn === undefined || state.typedJoins.length === 0) return undefined
+	const proven = new Map<string, Set<string>>()
+	for (const condition of state.selectWhereFn(makeAccessor(state))) {
+		if (condition == null) continue
+		for (const fact of notNullFactsOf(condition)) {
+			const columns = proven.get(fact.source) ?? new Set<string>()
+			columns.add(fact.column)
+			proven.set(fact.source, columns)
+		}
+	}
+	return proven.size === 0 ? undefined : proven
+}
+
+/** A column type proven not NULL: a `Nullable` column's inner type, or its schema without the `null` member. */
+function nonNullColumn(type: CHType<string, any, any>): CHType<string, any, any> {
+	if (type._tag === "Nullable" && type.element !== undefined) {
+		return type.sqlName === undefined ? type.element : { ...type.element, sqlName: type.sqlName }
+	}
+	const schema = withoutNull(type.schema)
+	return schema === undefined
+		? type
+		: { ...type, schema, literalSchema: withoutNull(type.literalSchema) ?? type.literalSchema }
+}
+
 /**
  * A join's columns: its table's, or — for a joined subquery — the schemas of
  * the SELECT it will compile to.
  *
  * `innerJoinQuery` records no columns (there is no table to read them from), so
  * without this every `$.alias.field` off a joined subquery is untyped.
  */
-function joinColumnsOf(join: {
-	readonly type: "INNER" | "LEFT" | "CROSS"
-	readonly columns?: ColumnDefs
-	readonly innerQuery?: CHQuery<any, any, any>
-}): ColumnDefs | undefined {
+function joinColumnsOf(
+	join: {
+		readonly type: "INNER" | "LEFT" | "CROSS"
+		readonly columns?: ColumnDefs
+		readonly innerQuery?: CHQuery<any, any, any>
+	},
+	/** The columns a WHERE condition proves not NULL: they lose `null`, the LEFT JOIN's included. */
+	notNull?: ReadonlySet<string>,
+): ColumnDefs | undefined {
 	const exprs = join.innerQuery === undefined ? undefined : selectExprsOf(join.innerQuery)
 	const columns = join.columns ?? (exprs === undefined ? undefined : synthesizeColumns(exprs))
-	if (columns === undefined || join.type !== "LEFT") return columns
+	if (columns === undefined || (join.type !== "LEFT" && notNull === undefined)) return columns
 	return Object.fromEntries(
 		Object.entries(columns).map(([name, type]) => [
 			name,
-			{
-				...type,
-				schema: Schema.NullOr(type.schema),
-				literalSchema: Schema.NullOr(type.literalSchema),
-			},
+			notNull?.has(name) === true
+				? nonNullColumn(type)
+				: join.type !== "LEFT"
+					? type
+					: {
+							...type,
+							schema: Schema.NullOr(type.schema),
+							literalSchema: Schema.NullOr(type.literalSchema),
+						},
 		]),
 	)
 }
@@ -1298,6 +1341,8 @@ function selectExprsOf(query: CHQuery<any, any, any>): Record<string, unknown> |
  * bare `select()` over a subquery or union, every alias that source selects.
  */
 function selectRecordOf(state: CHQueryState, $: any): Record<string, any> | undefined {
+	const notNull = selectNotNullOf(state)
+	if (notNull !== undefined) $ = makeAccessor(state, notNull)
 	if (state.selectAll) {
 		const source = state.fromUnion ? unionExprsOf(state.fromUnion._state.queries) : state.fromQuery ? selectExprsOf(state.fromQuery) : undefined
 		return source === undefined ? undefined : Object.fromEntries(Object.keys(source).map((alias) => [alias, $[alias]]))

src/ch/expr.ts

@@ -16,6 +16,7 @@ import { encodeColumnLiteral, encodeLiteral, isSubnormal, numberText, sqlLiteral
 import { QueryBuilderDefect, QueryBuilderError } from "./errors"
 import { fail } from "./failure"
 import { markTenantColumn, markTenantPredicate, tenantColumnOf, tenantPredicatesOf } from "./tenant"
+import { markNotNullFacts, notNullFactsOf, type NotNullFact } from "./facts"
 
 // Core interfaces
 
@@ -252,10 +253,26 @@ export type Quotient<L, R> = [R] extends [number]
 			: number | Extract<L, null>
 	: number | null
 
-export interface ColumnRef<Name extends string, ColType extends CHType<string, any>> extends Expr<
+/**
+ * A column of a joined source, `Source` its alias. `isNotNull()` and `eq(…)`
+ * on it prove the column not NULL: the condition carries a `NotNullFact`, and a
+ * WHERE list that holds it narrows the column for a later `select`. The FROM
+ * source's columns have no `Source` and carry no fact.
+ */
+export interface ColumnRef<Name extends string, ColType extends CHType<string, any>, Source extends string = never> extends Expr<
 	InferTS<ColType>
 > {
 	readonly columnName: Name
+	/**
+	 * phantom: the column's type. Keeps a `ColumnRef` of one column type from
+	 * passing for another when the compiler compares refs member by member, as
+	 * it does for a `Source` it cannot measure (`get` needs a Map column).
+	 */
+	readonly _columnType?: ColType
+	/** `column = other`. True only for a non-NULL column, so it proves one. */
+	eq<Q = never>(other: Operand<InferTS<ColType>, Q>): Condition<Q, NotNullFacts<Source, Name>>
+	/** `column IS NOT NULL`. */
+	isNotNull(): Condition<never, NotNullFacts<Source, Name>>
 	/**
 	 * Access a key in a Map column: `$.Attrs.get("http.method")`.
 	 *
@@ -276,15 +293,51 @@ export interface ColumnRef<Name extends string, ColType extends CHType<string, a
  */
 export type MapValueOf<ColType> = [ColType] extends [CHType<"Map", Record<string, infer V>, any>] ? V : string
 
-export interface Condition<P = never> {
+export interface Condition<P = never, F extends {} = {}> {
 	readonly _brand: "Condition"
 	/** phantom: the `param.*` placeholders inside this condition. */
 	readonly _params?: (entries: P) => void
+	/**
+	 * phantom: the joined columns this condition proves not NULL, as
+	 * `{ alias: { column: true } }` (see `NotNullFacts`). An object, so a
+	 * condition can drop a fact on assignment but never gain one.
+	 */
+	readonly _facts?: F
 	toFragment(): SqlFragment
-	and<Q = never>(other: Condition<Q>): Condition<P | Q>
+	/** Both conditions hold, so the facts of both do. */
+	and<Q = never, G extends {} = {}>(other: Condition<Q, G>): Condition<P | Q, F & G>
+	/** Either condition may hold: no fact survives. */
 	or<Q = never>(other: Condition<Q>): Condition<P | Q>
 }
 
+/**
+ * The columns proven not NULL: `{ alias: { column: true } }`. `{}` when the
+ * source is `never`, which is how a FROM column proves nothing.
+ */
+export type NotNullFacts<Source extends string, Column extends string> = {
+	readonly [S in Source]: { readonly [C in Column]: true }
+}
+
+/** The facts a condition proves, `{}` for none. */
+export type FactsIn<X> = 0 extends 1 & X
+	? {}
+	: X extends { readonly _facts?: infer F }
+		? 0 extends 1 & F
+			? {}
+			: Exclude<F, undefined>
+		: {}
+
+/**
+ * The facts every condition of a list proves, when all of them are present.
+ * A list of unknown length may be empty, and a member typed `| undefined` may
+ * be skipped, so neither proves anything.
+ */
+export type AllFacts<C extends ReadonlyArray<unknown>> = number extends C["length"]
+	? {}
+	: FactsIntersection<{ readonly [K in keyof C]: undefined extends C[K] ? {} : FactsIn<C[K]> }[number]>
+
+type FactsIntersection<U> = (U extends unknown ? (u: U) => void : never) extends (i: infer I) => void ? I & {} : {}
+
 // Core helpers (exported for define-fn.ts and consumer extensibility)
 
 /** An already-built expression or condition, rather than a value to encode. */
@@ -638,6 +691,11 @@ export function makeColumnRef<Name extends string, ColType extends CHType<string
 	tenantColumn?: string,
 	/** The column's declared type, whose schema decodes its wire value. */
 	columnType?: ColType,
+	/**
+	 * The join alias the column belongs to. Set for a joined source's columns:
+	 * their `isNotNull()` and `eq(…)` record a `NotNullFact` (see `./facts`).
+	 */
+	factSource?: string,
 ): ColumnRef<Name, ColType> {
 	// `alias.Column` when qualified: the qualifier is quoted segment by segment,
 	// the column as one identifier (a ClickHouse `Nested` column has a dot).
@@ -673,7 +731,7 @@ export function makeColumnRef<Name extends string, ColType extends CHType<string
 			? toFragment(value)
 			: known(() => encodeColumnLiteral(columnType, value, columnName ?? name))
 	}
-	return Object.assign(
+	const ref = Object.assign(
 		base,
 		isTenantColumn
 			? {
@@ -707,18 +765,29 @@ export function makeColumnRef<Name extends string, ColType extends CHType<string
 			},
 		},
 	) as ColumnRef<Name, ColType>
+	if (factSource === undefined) return ref
+	// `x = y` and `x IS NOT NULL` are never true for a NULL `x`.
+	const fact: ReadonlyArray<NotNullFact> = [{ source: factSource, column: columnName ?? name }]
+	const { eq, isNotNull } = ref
+	return Object.assign(ref, {
+		eq: (other: any) => markNotNullFacts(eq(other), fact),
+		isNotNull: () => markNotNullFacts(isNotNull(), fact),
+	})
 }
 
 // Condition implementation
 
-export function makeCond(fragment: SqlFragment): Condition<any> {
+export function makeCond(fragment: SqlFragment): Condition<any, any> {
 	return {
 		_brand: "Condition" as const,
 		toFragment: () => fragment,
 		and(other) {
-			return markTenantPredicate(
-				makeCond(known(() => `(${compile(fragment)} AND ${compile(other.toFragment())})`)),
-				[...tenantPredicatesOf(this), ...tenantPredicatesOf(other)],
+			return markNotNullFacts(
+				markTenantPredicate(
+					makeCond(known(() => `(${compile(fragment)} AND ${compile(other.toFragment())})`)),
+					[...tenantPredicatesOf(this), ...tenantPredicatesOf(other)],
+				),
+				[...notNullFactsOf(this), ...notNullFactsOf(other)],
 			)
 		},
 		or: (other) => makeCond(known(() => `(${compile(fragment)} OR ${compile(other.toFragment())})`)),
@@ -777,18 +846,21 @@ export function notInList(expr: Expr<string>, values: readonly string[]): Condit
 /**
  * Conditions AND-joined, an `undefined` one skipped: `and(a, when(x, f), b)`.
  * With none left it is `undefined`, which a `where` list skips in turn. Tenant
- * evidence carries through, as with `.and`.
+ * evidence and not-NULL facts carry through, as with `.and`.
  */
-export function and<const C extends ReadonlyArray<Condition>>(...conditions: C): Condition<ParamsIn<C[number]>>
+export function and<const C extends ReadonlyArray<Condition>>(...conditions: C): Condition<ParamsIn<C[number]>, AllFacts<C>>
 export function and<const C extends ReadonlyArray<Condition | undefined>>(
 	...conditions: C
 ): Condition<ParamsIn<C[number]>> | undefined
 export function and(...conditions: ReadonlyArray<Condition | undefined>): Condition | undefined {
 	const present = conditions.filter((c): c is Condition => c !== undefined)
 	if (present.length <= 1) return present[0]
-	return markTenantPredicate(
-		makeCond(known(() => `(${present.map((c) => compile(c.toFragment())).join(" AND ")})`)),
-		present.flatMap((c) => tenantPredicatesOf(c)),
+	return markNotNullFacts(
+		markTenantPredicate(
+			makeCond(known(() => `(${present.map((c) => compile(c.toFragment())).join(" AND ")})`)),
+			present.flatMap((c) => tenantPredicatesOf(c)),
+		),
+		present.flatMap((c) => notNullFactsOf(c)),
 	)
 }
 

src/ch/facts.ts

@@ -0,0 +1,31 @@
+// Predicate facts
+//
+// A condition in a query's top-level WHERE list is true for every row the
+// query returns. Some conditions prove more than their own truth: `x IS NOT
+// NULL` and `x = y` are only true when `x` is not NULL. A *fact* records that
+// proof for one column of a joined source, so the builder can type and decode
+// that column without `| null` once the WHERE list holds it.
+//
+// The type side rides on `Condition`'s second type parameter; this module is
+// the runtime side, which `compile` reads to narrow the row schema the same way.
+// Only an AND keeps a fact: `.and`, `and(…)` and the WHERE list itself. An OR,
+// a NOT, or an optional `when` branch proves nothing about every row.
+
+import type { Condition } from "./expr"
+
+/** A joined column proven not NULL: the join alias and the column key. */
+export interface NotNullFact<Source extends string = string, Column extends string = string> {
+	readonly source: Source
+	readonly column: Column
+}
+
+const facts = new WeakMap<Condition, ReadonlyArray<NotNullFact>>()
+
+/** The not-NULL facts a condition proves; none for one that proves nothing. */
+export const notNullFactsOf = (condition: Condition): ReadonlyArray<NotNullFact> => facts.get(condition) ?? []
+
+/** Record what `condition` proves. Returns it, for use in an expression. */
+export const markNotNullFacts = <A extends Condition>(condition: A, proven: ReadonlyArray<NotNullFact>): A => {
+	if (proven.length > 0) facts.set(condition, proven)
+	return condition
+}