From 459f90b1e9edec7859d5a8d5a531f749a826d183 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 1/9] feat(database): defineRelations and findMany, one statement per level of with --- src/database.ts | 15 ++ src/database/relations.test-d.ts | 82 ++++++++ src/database/relations.test.ts | 192 +++++++++++++++++++ src/database/relations.ts | 293 +++++++++++++++++++++++++++++ tests/relations.clickhouse.test.ts | 72 +++++++ 5 files changed, 654 insertions(+) create mode 100644 src/database/relations.test-d.ts create mode 100644 src/database/relations.test.ts create mode 100644 src/database/relations.ts create mode 100644 tests/relations.clickhouse.test.ts diff --git a/src/database.ts b/src/database.ts index ba66164..7b9b94a 100644 --- a/src/database.ts +++ b/src/database.ts @@ -38,6 +38,21 @@ export { TransactionUnsupported, type TransactionError, } from "./database/errors" +export { + defineRelations, + findMany, + type FindManyOptions, + type FindManyRow, + type KeyColumn, + type Relation, + type RelationalSchema, + type RelationalTable, + type RelationBuilder, + type RelationOptions, + type RelationsConfig, + type RelationTables, + type WithOf, +} from "./database/relations" export { QuerySummary, withQuerySummary } from "./database/tracing" export { sql, type SqlIdentifier, type SqlRawText, type SqlTemplate } from "./database/sql" export type { DialectTransactions, IsolationLevel, TransactionSettings } from "./ch/dialect" diff --git a/src/database/relations.test-d.ts b/src/database/relations.test-d.ts new file mode 100644 index 0000000..05aa0eb --- /dev/null +++ b/src/database/relations.test-d.ts @@ -0,0 +1,82 @@ +// Type-level tests: relation declarations are checked against the tables, and +// `findMany` rows follow the `with` tree. + +import type { DateTime, Effect } from "effect" +import { expectTypeOf } from "expect-type" +import * as PG from "../postgres" +import * as Db from "../database" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.int4, title: PG.text, at: PG.timestamptz }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +type User = { readonly id: number; readonly name: string } +type FindError = Db.DatabaseError | import("../postgres").QueryBuilderError | import("../postgres").CompiledQueryDecodeError + +// No `with`: the table's own row. +expectTypeOf(Db.findMany(schema.users)).toEqualTypeOf, FindError, Db.Database>>() +expectTypeOf(Db.findMany(schema.users, { where: ($) => [$.name.eq("ada")] })).toEqualTypeOf< + Effect.Effect, FindError, Db.Database> +>() + +// `many` is an array, `one` is a row or null, nested to any depth. +expectTypeOf( + Db.findMany(schema.users, { + with: { posts: { where: ($) => [$.title.eq("a")], with: { author: true } } }, + }), +).toEqualTypeOf< + Effect.Effect< + ReadonlyArray<{ + readonly id: number + readonly name: string + readonly posts: ReadonlyArray<{ + readonly id: number + readonly authorId: number + readonly title: string + readonly at: DateTime.Utc + readonly author: User | null + }> + }>, + FindError, + Db.Database + > +>() + +// `where` and `orderBy` on a nested level see that level's columns. +Db.findMany(schema.users, { + // @ts-expect-error -- `title` is a column of posts, not users + where: ($) => [$.title.eq("a")], +}) + +// A relation name `with` does not declare. +// @ts-expect-error -- users has no relation `author` +Db.findMany(schema.users, { with: { author: true } }) + +// Declarations are checked: target table, `to` column, `from` column. +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- no table `comments` + users: { posts: r.many("comments", { from: "id", to: "authorId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- posts has no column `userId` + users: { posts: r.many("posts", { from: "id", to: "userId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- users has no column `uid` + users: { posts: r.many("posts", { from: "uid", to: "authorId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- a timestamptz decodes to DateTime, which is not a key + posts: { author: r.one("users", { from: "at", to: "id" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- `name` is already a column of users + users: { name: r.many("posts", { from: "id", to: "authorId" }) }, +})) diff --git a/src/database/relations.test.ts b/src/database/relations.test.ts new file mode 100644 index 0000000..59358ac --- /dev/null +++ b/src/database/relations.test.ts @@ -0,0 +1,192 @@ +// Relational reads (`defineRelations`, `findMany`) on PGlite: one statement per +// level of `with`, rows decoded through each level's own row codec. + +import { PgliteClient } from "@effect/sql-pglite" +import { assert, expect, layer } from "@effect/vitest" +import { Cause, DateTime, Effect, Exit, Layer } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as PG from "../postgres" +import { postgresDialect } from "../pg/dialect" +import * as Db from "../database" + +const statements: Array = [] + +const Live = Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), +).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), title: PG.text }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +const setup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob'), (3, 'cy')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'a1'), (11, 1, 'a2'), (12, 2, 'b1')`) +}) + +layer(Live, { excludeTestServices: true })("findMany on PGlite", (it) => { + it.effect("loads a many relation and a nested one, one statement per level", () => + Effect.gen(function* () { + yield* setup + statements.length = 0 + const users = yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { orderBy: ($) => [[$.id, "asc"]], with: { author: true } } }, + }) + expect(users).toEqual([ + { + id: 1, + name: "ada", + posts: [ + { id: 10, authorId: 1, title: "a1", author: { id: 1, name: "ada" } }, + { id: 11, authorId: 1, title: "a2", author: { id: 1, name: "ada" } }, + ], + }, + { id: 2, name: "bob", posts: [{ id: 12, authorId: 2, title: "b1", author: { id: 2, name: "bob" } }] }, + { id: 3, name: "cy", posts: [] }, + ]) + expect(statements).toHaveLength(3) + expect(statements[1]).toMatch(/IN \(/) + }), + ) + + it.effect("a one relation is null when no row matches, and an empty level sends nothing", () => + Effect.gen(function* () { + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (13, 99, 'orphan')`) + const orphan = yield* Db.findMany(schema.posts, { where: ($) => [$.id.eq(13)], with: { author: true } }) + expect(orphan).toEqual([{ id: 13, authorId: 99, title: "orphan", author: null }]) + statements.length = 0 + const none = yield* Db.findMany(schema.users, { where: ($) => [$.id.eq(404)], with: { posts: true } }) + expect(none).toEqual([]) + expect(statements).toHaveLength(1) + }), + ) + + it.effect("a nested where filters that level only", () => + Effect.gen(function* () { + const users = yield* Db.findMany(schema.users, { + where: ($) => [$.id.in_(1, 2)], + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { where: ($) => [$.title.eq("a2")] } }, + }) + expect(users.map((user) => [user.name, user.posts.map((post) => post.title)])).toEqual([ + ["ada", ["a2"]], + ["bob", []], + ]) + }), + ) + + it.effect("nested rows decode through their column codecs", () => + Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE events (id int4 PRIMARY KEY, user_id int4 NOT NULL, at timestamptz NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO events VALUES (1, 1, '2026-01-02T03:04:05Z')`) + const Events = PG.table("events", { + columns: { id: PG.int4, userId: PG.column(PG.int4, { name: "user_id" }), at: PG.timestamptz }, + primaryKey: ["id"], + }) + const withEvents = Db.defineRelations({ users: Users, events: Events }, (r) => ({ + users: { events: r.many("events", { from: "id", to: "userId" }) }, + })) + const [ada] = yield* Db.findMany(withEvents.users, { where: ($) => [$.id.eq(1)], with: { events: true } }) + const at = ada!.events[0]!.at + assert(DateTime.isDateTime(at)) + expect(DateTime.formatIso(at)).toBe("2026-01-02T03:04:05.000Z") + }), + ) +}) + +// Tenant scope on every level: a separate database that refuses unscoped statements. +const Orgs = PG.table("orgs", { columns: { id: PG.text }, primaryKey: ["id"] }) +const Units = PG.table("units", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const Locks = PG.table("locks", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }), unitId: PG.column(PG.int4, { name: "unit_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const tenancy = Db.defineRelations({ orgs: Orgs, units: Units, locks: Locks }, (r) => ({ + orgs: { units: r.many("units", { from: "id", to: "orgId" }) }, + units: { locks: r.many("locks", { from: "id", to: "unitId" }) }, +})) + +const Scoped = Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), +).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const tenantSetup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE orgs (id text PRIMARY KEY)`) + yield* Db.execute(Db.sql`CREATE TABLE units (id int4 PRIMARY KEY, org_id text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE locks (id int4 PRIMARY KEY, org_id text NOT NULL, unit_id int4 NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO orgs VALUES ('a'), ('b')`) + yield* Db.execute(Db.sql`INSERT INTO units VALUES (1, 'a'), (2, 'b')`) + // Lock 21 points at unit 1 but belongs to org b: bad data a tenant pin must not leak. + yield* Db.execute(Db.sql`INSERT INTO locks VALUES (10, 'a', 1), (20, 'b', 2), (21, 'b', 1)`) +}) + +layer(Scoped, { excludeTestServices: true })("findMany tenant scope", (it) => { + it.effect("a level under one tenant's rows is pinned to that tenant", () => + Effect.gen(function* () { + yield* tenantSetup + statements.length = 0 + const units = yield* Db.findMany(tenancy.units, { where: ($) => [$.orgId.eq("a")], with: { locks: true } }) + expect(units).toEqual([{ id: 1, orgId: "a", locks: [{ id: 10, orgId: "a", unitId: 1 }] }]) + expect(statements[1]).toMatch(/org_id" = /) + }), + ) + + it.effect("an unscoped top level is refused before any nested level runs", () => + Effect.gen(function* () { + statements.length = 0 + const exit = yield* Effect.exit(Db.findMany(tenancy.units, { with: { locks: true } })) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + expect(statements).toEqual([]) + }), + ) + + it.effect("an untenanted parent gives a tenanted level nothing to pin, so it is refused", () => + Effect.gen(function* () { + const exit = yield* Effect.exit(Db.findMany(tenancy.orgs, { with: { units: true } })) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + }), + ) + + it.effect("crossTenant declares every level", () => + Effect.gen(function* () { + const units = yield* Db.findMany(tenancy.units, { + orderBy: ($) => [[$.id, "asc"]], + with: { locks: { orderBy: ($) => [[$.id, "asc"]] } }, + crossTenant: true, + }) + expect(units.map((unit) => unit.locks.map((lock) => lock.id))).toEqual([[10, 21], [20]]) + }), + ) +}) diff --git a/src/database/relations.ts b/src/database/relations.ts new file mode 100644 index 0000000..d5fae05 --- /dev/null +++ b/src/database/relations.ts @@ -0,0 +1,293 @@ +// Relational reads: relations declared over tables, and `findMany` with a +// nested `with`, as Drizzle's relational queries and Prisma's `include`. +// +// Each level of `with` is one ordinary builder query, `WHERE IN (...)` +// over the keys the level above returned. So every level compiles, decodes and +// derives its tenant scope the way any other query does, and `run` checks it. +// See design/relations.md for why this and not JSON aggregation. + +import { Effect } from "effect" +import type { CompiledQueryDecodeError } from "../ch/compile" +import type { QueryBuilderError } from "../ch/errors" +import type { Condition, Expr } from "../ch/expr" +import { from, type ColumnAccessor, type CHQuery } from "../ch/query" +import type { SelectRowOf, Table } from "../ch/table" +import type { ColumnDefs, InferTS } from "../ch/types" +import { Database } from "./database" +import type { DatabaseError } from "./errors" + +type AnyTable = Table + +/** The tables a relation schema covers, by the key relations use to name them. */ +export type RelationTables = Record + +type ColumnsOf = T extends Table ? Cols : never + +/** + * Columns a relation can join on: their decoded value is a string, number or + * bigint, which the loader matches by value. A `DateTime` or bytes column is + * not a key. + */ +export type KeyColumn = { + [K in keyof ColumnsOf]: NonNullable[K]>> extends string | number | bigint ? K : never +}[keyof ColumnsOf] & + string + +/** + * A relation from one table to another: rows of `target` whose `to` column + * equals this row's `from` column. `"one"` loads the first such row or `null`, + * `"many"` loads all of them. + */ +export interface Relation< + Kind extends "one" | "many" = "one" | "many", + Target extends string = string, + From extends string = string, + To extends string = string, +> { + readonly _tag: "Relation" + readonly kind: Kind + readonly target: Target + readonly from: From + readonly to: To +} + +/** What the `defineRelations` callback gets: `r.one` and `r.many`, checked against the tables. */ +export interface RelationBuilder { + readonly one: >( + target: Target, + on: { readonly from: From; readonly to: To }, + ) => Relation<"one", Target, From, To> + readonly many: >( + target: Target, + on: { readonly from: From; readonly to: To }, + ) => Relation<"many", Target, From, To> +} + +/** + * The relations of each table, by relation name. `from` must be a key column + * of the source table, and a relation name must not be one of its columns: the + * loaded rows carry both. + */ +export type RelationsConfig = { + readonly [S in keyof Tables]?: { readonly [name: string]: Relation<"one" | "many", keyof Tables & string, KeyColumn> } & { + readonly [C in keyof ColumnsOf]?: never + } +} + +type RelationsOf = K extends keyof R ? NonNullable : {} + +/** One table of a relation schema, as `findMany` takes it: `schema.users`. */ +export interface RelationalTable { + readonly _tag: "RelationalTable" + readonly key: K + readonly table: Tables[K] + /** @internal */ + readonly tables: Tables + /** @internal */ + readonly relations: R +} + +/** What `defineRelations` returns: one `RelationalTable` per table key. */ +export type RelationalSchema = { + readonly [K in keyof Tables & string]: RelationalTable +} + +/** + * Relations over a set of tables, Drizzle's `defineRelations`. Relations name + * their target by its key in `tables`, so two tables can point at each other + * and a table at itself. + * + * ```ts + * const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + * users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + * posts: { author: r.one("users", { from: "authorId", to: "id" }) }, + * })) + * ``` + */ +export const defineRelations = >( + tables: Tables, + build: (r: RelationBuilder) => R, +): RelationalSchema => { + const relation = + (kind: "one" | "many") => + (target: string, on: { readonly from: string; readonly to: string }): Relation => ({ + _tag: "Relation", + kind, + target, + from: on.from, + to: on.to, + }) + const relations = build({ one: relation("one"), many: relation("many") } as unknown as RelationBuilder) + return Object.fromEntries( + Object.keys(tables).map((key) => [key, { _tag: "RelationalTable", key, table: tables[key], tables, relations }]), + ) as unknown as RelationalSchema +} + +/** Options for one nested level of `with`. */ +export interface RelationOptions { + /** More conditions on this level, ANDed with the relation's key match. */ + readonly where?: ($: ColumnAccessor>) => ReadonlyArray + /** The order of this level's rows, within each parent. */ + readonly orderBy?: ($: ColumnAccessor>) => ReadonlyArray, "asc" | "desc"]> + /** Relations to load under each row of this level. */ + readonly with?: WithOf +} + +/** `with`: each relation to load, as `true` or with its own options. */ +export type WithOf = { + readonly [N in keyof RelationsOf]?: + | true + | (RelationsOf[N] extends Relation ? RelationOptions : never) +} + +/** Options for `findMany`: the top level also takes `limit`, `offset` and `crossTenant`. */ +export interface FindManyOptions + extends RelationOptions { + readonly limit?: number + readonly offset?: number + /** Declare every level `.crossTenant()`; see docs/tenant-scoping.md. */ + readonly crossTenant?: boolean +} + +type WithIn = O extends { readonly with?: infer W } ? (W extends object ? W : {}) : {} + +type Simplify = { readonly [K in keyof A]: A[K] } & {} + +/** A row `findMany` returns: every column of the table, plus each relation `with` asked for. */ +export type FindManyRow = Simplify< + SelectRowOf & { + readonly [N in keyof W & keyof RelationsOf as W[N] extends undefined ? never : N]: RelationsOf[N] extends Relation< + infer Kind, + infer T extends keyof Tables & string + > + ? Kind extends "many" + ? ReadonlyArray>> + : FindManyRow> | null + : never + } +> + +type FindError = DatabaseError | QueryBuilderError | CompiledQueryDecodeError +type Row = Record +interface Level { + readonly where?: ($: any) => ReadonlyArray + readonly orderBy?: ($: any) => ReadonlyArray, "asc" | "desc"]> + readonly with?: Record + readonly limit?: number + readonly offset?: number +} + +const distinct = (values: ReadonlyArray): Array => [ + ...new Set(values.filter((value) => value !== null && value !== undefined)), +] + +const levelQuery = ( + table: AnyTable, + level: Level, + match: ($: any) => ReadonlyArray, + crossTenant: boolean, +): CHQuery => { + let query: CHQuery = from(table as Table) + .select() + .where(($: any) => [...match($), ...(level.where?.($) ?? [])]) + if (level.orderBy !== undefined) query = query.orderBy(level.orderBy as any) + if (level.limit !== undefined) query = query.limit(level.limit) + if (level.offset !== undefined) query = query.offset(level.offset) + return crossTenant ? query.crossTenant() : query +} + +/** + * Load the relations `level.with` asks for under `rows` of `table`: one query + * per relation, then the next level under its rows. + */ +const attach = ( + tables: RelationTables, + relations: Record | undefined>, + key: string, + rows: ReadonlyArray, + level: Level, + crossTenant: boolean, +): Effect.Effect, FindError, Database> => + Effect.gen(function* () { + const names = Object.entries(level.with ?? {}).filter(([, spec]) => spec !== undefined) + if (names.length === 0) return rows + const db = yield* Database + const parent = tables[key]! + const loaded: Record>> = {} + for (const [name, spec] of names) { + const relation = relations[key]?.[name] + if (relation === undefined) { + return yield* Effect.die(new Error(`findMany: ${key} has no relation named ${name}`)) + } + const target = tables[relation.target]! + const nested: Level = spec === true ? {} : (spec as Level) + const keys = distinct(rows.map((row) => row[relation.from])) + let children: ReadonlyArray = [] + if (keys.length > 0) { + // The child level keeps the tenant its parents were read for: when the + // parent rows all carry one tenant, the child query pins its own tenant + // column to it, so its scope is proved like any other query's. + const tenants = + parent.tenantColumn !== undefined && target.tenantColumn !== undefined + ? distinct(rows.map((row) => row[parent.tenantColumn!])) + : [] + const tenantColumn = target.tenantColumn + const query = levelQuery( + target, + nested, + ($) => [ + $[relation.to].in_(...keys), + tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, + ], + crossTenant, + ) + const decoded = (yield* db.run(query as any)) as ReadonlyArray + children = yield* attach(tables, relations, relation.target, decoded, nested, crossTenant) + } + const byKey = new Map>() + for (const child of children) { + const value = child[relation.to] + const group = byKey.get(value) + if (group === undefined) byKey.set(value, [child]) + else group.push(child) + } + loaded[name] = byKey + } + return rows.map((row) => { + const out: Row = { ...row } + for (const [name] of names) { + const relation = relations[key]![name]! + const group = loaded[name]!.get(row[relation.from]) ?? [] + out[name] = relation.kind === "many" ? group : (group[0] ?? null) + } + return out + }) + }) + +/** + * Read rows of one table with the relations `with` names, nested to any depth. + * Each level is one statement, `WHERE IN (...)` over the keys of the + * level above, decoded through its own row codec. A level whose parents carry + * one tenant is pinned to it. Levels are separate statements: run `findMany` + * inside `transaction` for one snapshot across them. + * + * ```ts + * Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) + * ``` + */ +export const findMany: { + ( + table: RelationalTable, + ): Effect.Effect>, FindError, Database> + >( + table: RelationalTable, + options: O, + ): Effect.Effect>>, FindError, Database> +} = (table: RelationalTable, options?: FindManyOptions) => + Effect.gen(function* () { + const level = (options ?? {}) as Level & { readonly crossTenant?: boolean } + const crossTenant = level.crossTenant === true + const db = yield* Database + const rows = (yield* db.run(levelQuery(table.table, level, () => [], crossTenant) as any)) as ReadonlyArray + return yield* attach(table.tables, table.relations as any, table.key, rows, level, crossTenant) + }) as any diff --git a/tests/relations.clickhouse.test.ts b/tests/relations.clickhouse.test.ts new file mode 100644 index 0000000..98cd43c --- /dev/null +++ b/tests/relations.clickhouse.test.ts @@ -0,0 +1,72 @@ +// `findMany` on a live ClickHouse: relations need no foreign keys, so the same +// loader runs here. Creates two tables in a throwaway database. + +import { ClickhouseClient } from "@effect/sql-clickhouse" +import { Effect } from "effect" +import { describe, expect, it } from "vitest" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as Db from "@maple-dev/effect-orm/database" +import { endpoint } from "./clickhouse-support" + +const user = process.env.EFFECT_ORM_CLICKHOUSE_USER ?? "default" +const password = process.env.EFFECT_ORM_CLICKHOUSE_PASSWORD ?? "" + +const Services = CH.table("services", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, Id: CH.uint32, Name: CH.string }, +}) +const Events = CH.table("events", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, ServiceId: CH.uint32, At: CH.dateTime64 }, +}) +const schema = Db.defineRelations({ services: Services, events: Events }, (r) => ({ + services: { events: r.many("events", { from: "Id", to: "ServiceId" }) }, + events: { service: r.one("services", { from: "ServiceId", to: "Id" }) }, +})) + +describe.skipIf(!endpoint)("findMany on live ClickHouse", () => { + it("loads nested relations, pinned to the tenant, one statement per level", async () => { + const { rows, sent } = await Effect.runPromise( + Effect.gen(function* () { + const database = `eo_relations_${Date.now()}_${Math.floor(Math.random() * 1e6)}` + const admin = yield* ClickhouseClient.ClickhouseClient + yield* admin.asCommand(admin.unsafe(`CREATE DATABASE ${database}`)) + return yield* Effect.gen(function* () { + const client = yield* ClickhouseClient.ClickhouseClient + const sent: Array = [] + const db = Db.fromSqlClient(client, { + dialect: CH.clickhouseDialect, + command: client.asCommand, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void sent.push(statement.sql)), + }) + yield* db.execute(Db.sql`CREATE TABLE services (OrgId String, Id UInt32, Name String) ENGINE = MergeTree ORDER BY Id`) + yield* db.execute( + Db.sql`CREATE TABLE events (OrgId String, ServiceId UInt32, At DateTime64(3)) ENGINE = MergeTree ORDER BY At`, + ) + yield* db.execute(Db.sql`INSERT INTO services VALUES ('a', 1, 'api'), ('b', 2, 'web')`) + yield* db.execute( + Db.sql`INSERT INTO events VALUES ('a', 1, '2026-01-01 00:00:00.000'), ('b', 1, '2026-01-01 00:00:01.000')`, + ) + sent.length = 0 + const rows = yield* Db.findMany(schema.services, { + where: ($) => [$.OrgId.eq("a")], + with: { events: { with: { service: true } } }, + }).pipe(Effect.provideService(Db.Database, db)) + return { rows, sent } + }).pipe( + Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database })), + Effect.ensuring(Effect.orDie(admin.asCommand(admin.unsafe(`DROP DATABASE IF EXISTS ${database} SYNC`)))), + ) + }).pipe(Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database: "default" }))), + ) + expect(sent).toHaveLength(3) + expect(rows).toHaveLength(1) + const [api] = rows + expect(api!.Name).toBe("api") + expect(api!.events).toHaveLength(1) + expect(api!.events[0]!.service?.Name).toBe("api") + }) +}) From d08bef864e1b2c92528ba00bfca9c4111b316a2c Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 2/9] docs: relational reads with defineRelations and findMany --- README.md | 1 + docs/README.md | 1 + docs/reference.md | 1 + docs/relations.md | 129 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 docs/relations.md diff --git a/README.md b/README.md index 4863efe..afa3ed5 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,7 @@ Full guides live in [`docs/`](./docs/README.md): | [Decoding results](./docs/decoding-results.md) | `rowSchema`, `decodeRows`, decode errors | | [Running a query](./docs/running-queries.md) | Executing the SQL with a real client, wire settings, `SETTINGS` | | [Tenant scoping](./docs/tenant-scoping.md) | `tenantColumn`, what marks a query scoped, `crossTenant()` | +| [Relational reads](./docs/relations.md) | `defineRelations`, `findMany` with nested `with` | | [Postgres](./docs/postgres.md) | The Postgres dialect, its column types and functions | | [Schema and migrations](./docs/migrations.md) | DDL from `table`, `effect-orm generate`, applying migrations | | [Extending the DSL](./docs/extending.md) | `defineFn`, raw escape hatches, handwritten SQL | diff --git a/docs/README.md b/docs/README.md index 7a61d11..a150170 100644 --- a/docs/README.md +++ b/docs/README.md @@ -65,6 +65,7 @@ Roughly in reading order. | [Postgres](./postgres.md) | The Postgres dialect, its column types and functions | | [Schema and migrations](./migrations.md) | DDL from `CH.table` / `PG.table`, `materializedView`, `effect-orm generate`, applying migrations, adopting drizzle-kit | | [Statements and transactions](./database.md) | `Database` over your `SqlClient`: `run`, `execute`, `transaction`, retry | +| [Relational reads](./relations.md) | `defineRelations`, `findMany` with nested `with`, tenant scope per level | ## Reference diff --git a/docs/reference.md b/docs/reference.md index abc2d46..f34bc3c 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -513,6 +513,7 @@ placeholders, double-quoted identifiers. | `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | +| `defineRelations`, `findMany`, `Relation`, `RelationBuilder`, `RelationsConfig`, `RelationTables`, `RelationalSchema`, `RelationalTable`, `RelationOptions`, `FindManyOptions`, `FindManyRow`, `WithOf`, `KeyColumn` — relational reads, one statement per level ([Relational reads](./relations.md)) | `/database` | | `withQuerySummary`, `QuerySummary` — the `db.query.summary` of the statements an effect runs, for spans | `/database` | | `ClickhouseHttpClient`, `layer`, `make`, `ClickhouseHttpClientConfig` — a ClickHouse `SqlClient` over `fetch`, for Workers | `/clickhouse-http` | | `defineConfig`, `generate`, `check`, `pull`, `loadSchema`, `readMigrations`, `analyze`, `KitError` | `/kit` | diff --git a/docs/relations.md b/docs/relations.md new file mode 100644 index 0000000..173046e --- /dev/null +++ b/docs/relations.md @@ -0,0 +1,129 @@ +# Relational reads + +`Db.defineRelations` declares how tables relate. `Db.findMany` reads rows of one table with the +related rows under them, nested to any depth, as Drizzle's relational queries and Prisma's +`include` do. Both are in `@maple-dev/effect-orm/database`. + +```ts title="relations-find-many.ts" +import { PgliteClient } from "@effect/sql-pglite" +import { Effect, Layer } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), title: PG.text }, + primaryKey: ["id"], +}) + +export const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +const DatabaseLive = Db.layerSqlClient({ dialect: PG.postgresDialect }).pipe( + Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" })), +) + +export const users = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'hello')`) + return yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { with: { author: true } } }, + }) +}).pipe(Effect.provide(DatabaseLive)) +// [ +// { id: 1, name: "ada", posts: [{ id: 10, authorId: 1, title: "hello", author: { id: 1, name: "ada" } }] }, +// { id: 2, name: "bob", posts: [] }, +// ] +``` + +## Declare relations + +`defineRelations(tables, build)` takes the tables by key, and a callback that returns the +relations of each table by name. A relation names its target by that key, so two tables can point +at each other, and a table can point at itself. + +| Builder | Loads | Result | +| --- | --- | --- | +| `r.many(target, { from, to })` | Every `target` row whose `to` column equals this row's `from` column | `ReadonlyArray` | +| `r.one(target, { from, to })` | The first such row | `Row \| null` | + +The types check each declaration. The target must be a key of `tables`. `from` must be a column of +the source table and `to` a column of the target. Both must decode to a `string`, `number` or +`bigint`, because the loader matches keys by value. A relation name must not be a column name of +its table, because the loaded row carries both. + +A `one` relation is always `| null`. A row can be missing even when a foreign key says it cannot: +another statement can delete it between two levels. + +Relations are metadata only. They do not need a foreign key in the database, so they work on +ClickHouse too. + +_(Backed by `src/database/relations.test-d.ts`, and `tests/relations.clickhouse.test.ts` on ClickHouse.)_ + +## Read with `findMany` + +`findMany(schema.table, options?)` returns every column of the table, plus each relation that +`with` names. `with` takes `true`, or the options of that level: + +| Option | Levels | Meaning | +| --- | --- | --- | +| `where` | All | More conditions, ANDed with the relation's key match | +| `orderBy` | All | The order of the rows; nested rows keep it within each parent | +| `with` | All | Relations to load under each row of this level | +| `limit`, `offset` | Top only | Paging of the top level | +| `crossTenant` | Top only | Declare every level `.crossTenant()` | + +Rows come back decoded through the codecs of their columns at every level. A `timestamptz` in a +nested row is a `DateTime.Utc`, not a string. + +## One statement per level + +Each level of `with` is one ordinary builder query: every column of the target table, +`WHERE IN ()`, plus the level's own `where` and +`orderBy`. The loader groups the rows by key and puts them under their parents. A level with no +keys sends no statement. A tree with three levels sends three statements, whatever the number of +rows. + +The levels are separate statements. Outside a transaction, a write from another connection can +land between two of them. Run `findMany` inside [`Db.transaction`](./database.md#transactions) +with `isolationLevel: "repeatable read"` for one snapshot across all levels. + +All keys of a level go into one `IN` list. Postgres allows at most 65,535 bound values in one +statement, so keep the top level paged with `limit`. + +## Tenant scope on every level + +Each level runs through `run`, so each level has its own +[`tenantScope`](./tenant-scoping.md), and `requireTenantScope` checks each one. + +When the parent rows all carry one tenant, and both tables declare a `tenantColumn`, the nested +query also gets ` = `. That pins the level the same way a hand-written +filter does. It also hides a child row whose tenant differs from its parent's. + +A level gets no pin when: + +- the parent table has no `tenantColumn`; +- the parent rows carry more than one tenant. + +That level is then `"cross-tenant"`, and `requireTenantScope` refuses it. There is one exception: +if the relation's `to` column is the target's tenant column and the level has one key, the +one-value `IN` scopes it already. With `crossTenant: true`, every level is declared +cross-tenant and runs. + +_(Backed by `src/database/relations.test.ts`.)_ + +## Not yet + +- Choosing columns per level (`columns: { id: true }`). Every level reads every column. +- A `limit` per parent on a nested level. That needs a window function or a lateral join. +- Composite keys (`from` and `to` with more than one column). +- `findFirst`. Use `findMany` with `limit: 1`. +- Filters and counts over relations (`_count`, `some`, `every`, `none`). +- `param.*` markers in a `where`: `findMany` takes no params, so write values in. + +See [design/relations.md](../design/relations.md) for the design and the alternatives. From 1311ae81acd3c1145c41c3704a9dd6050cc22496 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 3/9] docs(design): relational reads, one statement per level --- .changeset/relational-reads.md | 5 ++ design/relations.md | 108 +++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 .changeset/relational-reads.md create mode 100644 design/relations.md diff --git a/.changeset/relational-reads.md b/.changeset/relational-reads.md new file mode 100644 index 0000000..a2858dd --- /dev/null +++ b/.changeset/relational-reads.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`Db.defineRelations(tables, (r) => ...)` and `Db.findMany(schema.table, { where, orderBy, limit, with })` in `/database` read rows with their related rows, nested to any depth, as Drizzle's relational queries and Prisma's `include` do. `r.one` and `r.many` are checked against the tables. Each level of `with` is one statement, `WHERE IN (...)` over the keys of the level above, decoded through its own row codec, so a nested `timestamptz` is a `DateTime.Utc`. A level whose parent rows carry one tenant is pinned to it, and `requireTenantScope` checks every level. See docs/relations.md and design/relations.md. diff --git a/design/relations.md b/design/relations.md new file mode 100644 index 0000000..a7f5377 --- /dev/null +++ b/design/relations.md @@ -0,0 +1,108 @@ +# Relational reads: `defineRelations` and `findMany` + +Status: first slice built (issue #79; `docs/relations.md`). `one` and `many` relations over +single-column keys, `with` to any depth, `where` and `orderBy` per level, `limit` and `offset` on +the top level. Section 5 lists what is not built. + +## 1. Goal + +Read a row with its related rows in one call, as Drizzle's relational queries +(`db.query.users.findMany({ with: { posts: true } })`) and Prisma's `include` do. Keep what the +rest of the library gives a query: + +- rows decoded through the column codecs, at every level; +- a tenant scope derived for every statement, which `requireTenantScope` checks; +- a result type derived from the `with` tree. + +## 2. The API + +```ts +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) +``` + +- **One callback over all tables**, as Drizzle v2 `defineRelations`. A relation names its target + by key, not by value, so tables that point at each other need no thunks. +- **Relations live beside the tables, not in them.** `table()` stays as it is, and the DDL, the + diff and the importers do not change. A Prisma importer can write the `defineRelations` call. +- **`findMany` lives in `/database`.** It runs more than one statement, so it needs the + `Database` service. `defineRelations` sits next to it, so the dialect entries do not change. +- **Checked in the types.** The target must be a table key, `to` a column of the target, `from` a + column of the source, and both must decode to `string | number | bigint`. A relation name must + not be a column name. + +## 3. One statement per level, not JSON aggregation + +The issue proposed one statement with JSON aggregation (`jsonb_agg` / `json_object`), as Drizzle +RQB v2 does. This slice runs **one statement per level** instead: +`SELECT FROM WHERE IN () [AND ] +[AND ] [ORDER BY ...]`. The loader then groups the rows by key in memory. Prisma does the +same by default. + +Why: + +1. **Decoding stays correct for free.** Each level is an ordinary builder query, decoded by its + own derived row codec. With JSON aggregation, every column comes back in its JSON form, not + its wire form. An `int8` becomes a JSON number and loses precision past 2^53, a `timestamptz` + becomes an ISO string in a different format, `bytea` becomes hex, and a `numeric` can change. + Each column type would need a second "from JSON" codec per dialect, or a cast to text in the + SQL. That is a large change to the type system for one feature. +2. **Tenant scope needs no new analysis.** Each level is a query the existing analysis already + understands. A nested JSON subquery would need correlated-subquery evidence through + `jsonb_agg`, which the analysis does not have yet. +3. **Every dialect.** The SQL is only `IN (...)`, so the same code runs on Postgres, on + ClickHouse, and later on SQLite / D1 and MySQL. JSON aggregation needs a different function + set per dialect, and ClickHouse has no `jsonb_agg`. +4. **The statement count is fixed.** It is the depth of the `with` tree, not the row count, so + there is no N+1. + +The costs: + +- **More round trips.** Three levels are three statements. On a server close to the app this is + small. JSON aggregation is better on a slow link. +- **No single snapshot.** Outside a transaction, a write can land between two levels. The docs + say to use `Db.transaction({ isolationLevel: "repeatable read" })`. `findMany` does not open a + transaction itself: ClickHouse has none, and the caller may already be in one. +- **Long `IN` lists.** All keys of a level are in one list. Postgres has a limit of 65,535 bound + values. Chunking belongs with #57 (parameter limits). + +JSON aggregation can come later as a second strategy behind the same API, per dialect, once the +column types have JSON codecs. The result type and the tenant rule would not change. + +## 4. Tenant scope on nested levels + +Each level goes through `Database.run`, so it is checked like any other statement. A nested +level filters on the parent keys, which says nothing about the tenant. So the loader adds a pin: + +> When the parent table and the target table both declare a `tenantColumn`, and every parent row +> carries the same tenant value `V`, the nested query also gets ` = V`. + +This is sound. The pin is an ordinary `eq` on a literal, so the existing analysis proves +`"single-tenant"`. `V` comes from rows the parent query read, so a level never reads a tenant its +parent did not read. The pin also hides bad data: a child row whose tenant differs from its +parent's. + +No pin when the parent table is untenanted, or the parent rows carry more than one tenant. The +level is then `"cross-tenant"`, and `requireTenantScope` refuses it, which is the safe default. A +level whose `to` column is the target's tenant column, with one key, is scoped by the one-value +`IN` already. `crossTenant: true` declares every level `.crossTenant()`. + +The pin does not pass through an untenanted level: a grandparent's tenant says nothing about a +grandchild reached through a shared table. + +## 5. Not built + +| What | Why not now | +| --- | --- | +| JSON aggregation strategy | Needs JSON codecs per column type (section 3) | +| `columns` per level | Small, but changes the row type of every level; next slice | +| `limit` per parent on a nested level | Needs `row_number()` over a partition, or a lateral join | +| Composite keys | The loader would match tuples; no issue asks for it yet | +| `findFirst` | `findMany` with `limit: 1`; cardinality helpers are #69 | +| `_count`, `some` / `every` / `none` | Issue #80 | +| Params in nested `where` | `findMany` takes no params; values go in as literals | +| SQLite, MySQL, D1 | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | From 098e3b8b10279cc05e501a18d42026cba1a8a12a Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:39:24 +0200 Subject: [PATCH 4/9] feat(database): relation filters some/none/every and relation counts Correlated EXISTS / NOT EXISTS conditions and a correlated count(*) over a declared relation, on the builder (Db.related) and in findMany's where. findMany's _count counts a many relation with one grouped statement per level. A subquery that correlates on the outer tenant column takes that column's tenant binding. --- src/ch/compile.ts | 22 +++- src/ch/tenant.ts | 16 +++ src/database.ts | 5 + src/database/relations.ts | 270 ++++++++++++++++++++++++++++++++------ 4 files changed, 270 insertions(+), 43 deletions(-) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..8ebd846 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -28,7 +28,7 @@ import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Di import { Effect, Option, Schema } from "effect" import { QueryBuilderDefect, QueryBuilderError } from "./errors" import { withSubqueryCompiler } from "./subquery-context" -import { tenantBoundOf, tenantColumnOf, tenantPredicatesOf, withTenantBound, type TenantPredicate } from "./tenant" +import { correlatedBound, correlatedTenantOf, tenantBoundOf, tenantColumnOf, tenantPredicatesOf, withTenantBound, type TenantPredicate } from "./tenant" // `QueryBuilderError` moved to ./errors so `expr.ts` can raise it too; still // exported from here, which is where every caller imports it from. @@ -741,6 +741,12 @@ function compileInner< * There is no reason to pass it by hand. */ enclosingCtes?: ReadonlyArray + /** + * Set for a correlated subquery that `markCorrelatedTenant` marked: its + * main table's tenant column is bound to this enclosing column, and the + * enclosing compile resolves that binding (see `tenant.ts`). + */ + correlatedTenant?: string }, ): CompiledQuery { // A nested query's columns are its own: nothing it renders reaches an @@ -843,6 +849,9 @@ function compileInner< fromFragment = raw(`(\n${body}\n) AS ${quoteIdent(state.fromQueryAlias ?? "")}`) } else { fromSource = sourceForTable(state.tableName, mainColumn) + if (options?.correlatedTenant !== undefined && fromSource.column !== undefined) { + fromSource = { ...fromSource, bound: correlatedBound(options.correlatedTenant) } + } fromFragment = mainAlias !== state.tableName ? raw(`${quoteIdentPath(state.tableName)} AS ${quoteIdentPath(mainAlias)}`) : identPath(state.tableName) @@ -856,13 +865,22 @@ function compileInner< sources.push({ scope: "cross-tenant" }) return subquery } + const correlatedTenant = correlatedTenantOf(subquery) const compiled = compileInner(subquery, params, { skipFormat: true, deferParams, nested: true, enclosingCtes: visibleCtes, + correlatedTenant, }) - sources.push(sourceOf(compiled)) + // A correlated subquery bound only through the enclosing tenant column + // reads the tenant that column is bound to: it is that column's source. + sources.push( + correlatedTenant !== undefined && compiled.tenantScope === "single-tenant" && + tenantBoundOf(compiled) === correlatedBound(correlatedTenant) + ? { column: correlatedTenant, scope: "cross-tenant" } + : sourceOf(compiled), + ) return compiled.sql }, () => { const joins = state.typedJoins.map((j) => { diff --git a/src/ch/tenant.ts b/src/ch/tenant.ts index b3a68aa..60a5f03 100644 --- a/src/ch/tenant.ts +++ b/src/ch/tenant.ts @@ -32,3 +32,19 @@ export const withTenantBound = (query: A, bound: string | if (bound !== undefined) bounds.set(query, bound) return query } + +/** + * A correlated subquery whose main table's tenant column equals a tenant column + * of the enclosing query (`inner.OrgId = outer.OrgId` in its own WHERE). Only + * the relation filters mark one, and they write that equality themselves. + * Compiled as a subquery, it takes the enclosing column's binding instead of + * needing one of its own; compiled alone, the mark means nothing. + */ +const correlated = new WeakMap() +export const correlatedTenantOf = (query: object): string | undefined => correlated.get(query) +export const markCorrelatedTenant = (query: A, outerColumn: string): A => { + correlated.set(query, outerColumn) + return query +} +/** The bound a correlated subquery reports: the outer column, not a value. */ +export const correlatedBound = (outerColumn: string): string => `\u0000correlated:${outerColumn}` diff --git a/src/database.ts b/src/database.ts index 7b9b94a..dca1bfc 100644 --- a/src/database.ts +++ b/src/database.ts @@ -41,6 +41,8 @@ export { export { defineRelations, findMany, + related, + type CountOf, type FindManyOptions, type FindManyRow, type KeyColumn, @@ -48,9 +50,12 @@ export { type RelationalSchema, type RelationalTable, type RelationBuilder, + type RelationFilter, type RelationOptions, type RelationsConfig, type RelationTables, + type RelationWhere, + type Related, type WithOf, } from "./database/relations" export { QuerySummary, withQuerySummary } from "./database/tracing" diff --git a/src/database/relations.ts b/src/database/relations.ts index d5fae05..f10d5b8 100644 --- a/src/database/relations.ts +++ b/src/database/relations.ts @@ -9,10 +9,15 @@ import { Effect } from "effect" import type { CompiledQueryDecodeError } from "../ch/compile" import type { QueryBuilderError } from "../ch/errors" -import type { Condition, Expr } from "../ch/expr" +import { and, lit, makeCond, makeExpr, not, type Condition, type Expr, type ParamsIn } from "../ch/expr" +import { builtins } from "../ch/functions/builtin" import { from, type ColumnAccessor, type CHQuery } from "../ch/query" +import { exists, subqueryExpr } from "../ch/subquery" import type { SelectRowOf, Table } from "../ch/table" -import type { ColumnDefs, InferTS } from "../ch/types" +import { markCorrelatedTenant, tenantColumnOf } from "../ch/tenant" +import { custom, type ColumnDefs, type InferTS } from "../ch/types" +import { PgNumber } from "../pg/types" +import { raw } from "../sql/sql-fragment" import { Database } from "./database" import type { DatabaseError } from "./errors" @@ -123,14 +128,138 @@ export const defineRelations = } +// Relation filters and counts +// +// `some`, `none` and `every` are correlated `EXISTS` / `NOT EXISTS` conditions, +// and `count` is a correlated `count(*)` subquery. The subquery reads the target +// table under its own alias, matches ` = `, and, when both +// tables declare a tenant column, ` = `. That last +// equality is what keeps the subquery inside the outer query's tenant: the +// compiler gives the subquery the outer column's binding (see `tenant.ts`). + +type TargetOf = N extends keyof RelationsOf + ? RelationsOf[N] extends Relation + ? T + : never + : never + +/** The conditions a relation filter adds to its subquery. */ +export type RelationWhere = ( + $: ColumnAccessor>, + r: Related, +) => ReadonlyArray + +type Conditions = ReadonlyArray + +/** The filters and the count of one relation, built on the outer query's `$`. */ +export interface RelationFilter { + /** At least one related row matches: `EXISTS (…)`. With no `where`, any related row. */ + readonly some: ( + where?: ($: ColumnAccessor>, r: Related) => C, + ) => Condition> + /** No related row matches: `NOT EXISTS (…)`. With no `where`, no related row at all. */ + readonly none: ( + where?: ($: ColumnAccessor>, r: Related) => C, + ) => Condition> + /** Every related row matches: `NOT EXISTS (… AND NOT (where))`. True when there is none. */ + readonly every: ( + where: ($: ColumnAccessor>, r: Related) => C, + ) => Condition> + /** The number of related rows that match, as a correlated `count(*)` subquery. */ + readonly count: ( + where?: ($: ColumnAccessor>, r: Related) => C, + ) => Expr> +} + +/** What `related` returns: one `RelationFilter` per relation of the table. */ +export type Related = { + readonly [N in keyof RelationsOf & string]: RelationFilter & keyof Tables & string> +} + +const aggregate = builtins("portable", "aggregate") +/** `count(*)` is the same in every dialect; a driver sends it as a number, a string or a bigint. */ +const countType = custom("Int64", PgNumber) +const countAll = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), countType.schema) +const always = (): Condition => makeCond(raw("1 = 1")) + +const relatedOf = ( + tables: RelationTables, + relations: Record | undefined>, + key: string, + $: any, +): Record> => + Object.fromEntries( + Object.entries(relations[key] ?? {}).map(([name, relation]) => { + const parent = tables[key]! + const target = tables[relation.target]! + const outerFrom = $[relation.from] + // `$` is qualified (`alias.column`): the subquery's alias extends it, so + // it never hides the outer source, even for a table related to itself. + const outerName: string = outerFrom.columnName + const alias = `${outerName.slice(0, outerName.length - relation.from.length - 1)}__${name}` + const outerTenant = + parent.tenantColumn !== undefined && target.tenantColumn !== undefined ? $[parent.tenantColumn] : undefined + const outerTenantColumn = outerTenant === undefined ? undefined : tenantColumnOf(outerTenant) + const inner = ( + where: (($: any, r: any) => ReadonlyArray) | undefined, + select: () => Record>, + negate = false, + ): CHQuery => { + const query = from(target as Table, alias) + .select(select as any) + .where((c: any) => { + const conditions = where?.(c, relatedOf(tables, relations, relation.target, c)) ?? [] + const own = negate ? [not(and(...conditions) ?? always())] : conditions + return [ + c[relation.to].eq(outerFrom), + outerTenant === undefined ? undefined : c[target.tenantColumn!].eq(outerTenant), + ...own, + ] + }) + return outerTenantColumn === undefined ? query : markCorrelatedTenant(query, outerTenantColumn) + } + const one = () => ({ one: lit(1) }) + const filter: RelationFilter = { + some: (where?: any) => exists(inner(where, one) as any), + none: (where?: any) => not(exists(inner(where, one) as any)), + every: (where: any) => not(exists(inner(where, one, true) as any)), + count: (where?: any) => + subqueryExpr(inner(where, () => ({ n: countAll() })) as any, countType, (sql) => `coalesce((${sql}), 0)`) as any, + } + return [name, filter] + }), + ) + +/** + * The relation filters and counts of a table, on the `$` of a builder query + * over that table: + * + * ```ts + * PG.from(Users) + * .select(($) => ({ id: $.id, posts: Db.related(schema.users, $).posts.count() })) + * .where(($) => [Db.related(schema.users, $).posts.some(($) => [$.published.eq(true)])]) + * ``` + * + * `findMany`'s `where` gets the same object as its second argument. + */ +export const related = ( + table: RelationalTable, + $: ColumnAccessor>, +): Related => relatedOf(table.tables, table.relations as any, table.key, $) as unknown as Related + /** Options for one nested level of `with`. */ export interface RelationOptions { - /** More conditions on this level, ANDed with the relation's key match. */ - readonly where?: ($: ColumnAccessor>) => ReadonlyArray + /** + * More conditions on this level, ANDed with the relation's key match. The + * second argument holds the relation filters of this level's table. + */ + readonly where?: RelationWhere /** The order of this level's rows, within each parent. */ readonly orderBy?: ($: ColumnAccessor>) => ReadonlyArray, "asc" | "desc"]> /** Relations to load under each row of this level. */ readonly with?: WithOf + /** Relations to count under each row of this level, Prisma's `_count`. */ + readonly _count?: CountOf } /** `with`: each relation to load, as `true` or with its own options. */ @@ -140,6 +269,13 @@ export type WithOf[N] extends Relation ? RelationOptions : never) } +/** `_count`: each `many` relation to count, as `true` or with a `where`. A `one` relation is refused. */ +export type CountOf = { + readonly [N in keyof RelationsOf]?: RelationsOf[N] extends Relation<"many", infer T extends keyof Tables & string> + ? true | { readonly where?: RelationWhere } + : never +} + /** Options for `findMany`: the top level also takes `limit`, `offset` and `crossTenant`. */ export interface FindManyOptions extends RelationOptions { @@ -150,59 +286,94 @@ export interface FindManyOptions = O extends { readonly with?: infer W } ? (W extends object ? W : {}) : {} +type CountIn = O extends { readonly _count?: infer C } ? (C extends object ? C : {}) : {} type Simplify = { readonly [K in keyof A]: A[K] } & {} -/** A row `findMany` returns: every column of the table, plus each relation `with` asked for. */ -export type FindManyRow = Simplify< +/** + * A row `findMany` returns: every column of the table, plus each relation + * `with` asked for, plus `_count` when the level counts relations. + */ +export type FindManyRow = Simplify< SelectRowOf & { readonly [N in keyof W & keyof RelationsOf as W[N] extends undefined ? never : N]: RelationsOf[N] extends Relation< infer Kind, infer T extends keyof Tables & string > ? Kind extends "many" - ? ReadonlyArray>> - : FindManyRow> | null + ? ReadonlyArray, CountIn>> + : FindManyRow, CountIn> | null : never - } + } & ([keyof C] extends [never] + ? {} + : { readonly _count: { readonly [N in keyof C as C[N] extends undefined ? never : N]: number } }) > type FindError = DatabaseError | QueryBuilderError | CompiledQueryDecodeError type Row = Record +type LevelWhere = ($: any, r: any) => ReadonlyArray interface Level { - readonly where?: ($: any) => ReadonlyArray + readonly where?: LevelWhere readonly orderBy?: ($: any) => ReadonlyArray, "asc" | "desc"]> readonly with?: Record + readonly _count?: Record readonly limit?: number readonly offset?: number } +type Relations = Record | undefined> const distinct = (values: ReadonlyArray): Array => [ ...new Set(values.filter((value) => value !== null && value !== undefined)), ] const levelQuery = ( - table: AnyTable, + tables: RelationTables, + relations: Relations, + key: string, level: Level, match: ($: any) => ReadonlyArray, crossTenant: boolean, ): CHQuery => { - let query: CHQuery = from(table as Table) + let query: CHQuery = from(tables[key] as Table) .select() - .where(($: any) => [...match($), ...(level.where?.($) ?? [])]) + .where(($: any) => [...match($), ...(level.where?.($, relatedOf(tables, relations, key, $)) ?? [])]) if (level.orderBy !== undefined) query = query.orderBy(level.orderBy as any) if (level.limit !== undefined) query = query.limit(level.limit) if (level.offset !== undefined) query = query.offset(level.offset) return crossTenant ? query.crossTenant() : query } +/** + * The key match of a nested level, ` IN ()`. The child level keeps + * the tenant its parents were read for: when the parent rows all carry one + * tenant, the child query pins its own tenant column to it, so its scope is + * proved like any other query's. + */ +const keyMatch = (parent: AnyTable, target: AnyTable, relation: Relation, rows: ReadonlyArray, keys: ReadonlyArray) => { + const tenants = + parent.tenantColumn !== undefined && target.tenantColumn !== undefined + ? distinct(rows.map((row) => row[parent.tenantColumn!])) + : [] + const tenantColumn = target.tenantColumn + return ($: any): ReadonlyArray => [ + $[relation.to].in_(...keys), + tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, + ] +} + +const relationOf = (relations: Relations, key: string, name: string): Effect.Effect => { + const relation = relations[key]?.[name] + return relation === undefined ? Effect.die(new Error(`findMany: ${key} has no relation named ${name}`)) : Effect.succeed(relation) +} + /** * Load the relations `level.with` asks for under `rows` of `table`: one query - * per relation, then the next level under its rows. + * per relation, then the next level under its rows. Then count the relations + * `level._count` asks for: one grouped query per relation. */ const attach = ( tables: RelationTables, - relations: Record | undefined>, + relations: Relations, key: string, rows: ReadonlyArray, level: Level, @@ -210,37 +381,19 @@ const attach = ( ): Effect.Effect, FindError, Database> => Effect.gen(function* () { const names = Object.entries(level.with ?? {}).filter(([, spec]) => spec !== undefined) - if (names.length === 0) return rows + const counted = Object.entries(level._count ?? {}).filter(([, spec]) => spec !== undefined) + if (names.length === 0 && counted.length === 0) return rows const db = yield* Database const parent = tables[key]! const loaded: Record>> = {} for (const [name, spec] of names) { - const relation = relations[key]?.[name] - if (relation === undefined) { - return yield* Effect.die(new Error(`findMany: ${key} has no relation named ${name}`)) - } + const relation = yield* relationOf(relations, key, name) const target = tables[relation.target]! const nested: Level = spec === true ? {} : (spec as Level) const keys = distinct(rows.map((row) => row[relation.from])) let children: ReadonlyArray = [] if (keys.length > 0) { - // The child level keeps the tenant its parents were read for: when the - // parent rows all carry one tenant, the child query pins its own tenant - // column to it, so its scope is proved like any other query's. - const tenants = - parent.tenantColumn !== undefined && target.tenantColumn !== undefined - ? distinct(rows.map((row) => row[parent.tenantColumn!])) - : [] - const tenantColumn = target.tenantColumn - const query = levelQuery( - target, - nested, - ($) => [ - $[relation.to].in_(...keys), - tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, - ], - crossTenant, - ) + const query = levelQuery(tables, relations, relation.target, nested, keyMatch(parent, target, relation, rows, keys), crossTenant) const decoded = (yield* db.run(query as any)) as ReadonlyArray children = yield* attach(tables, relations, relation.target, decoded, nested, crossTenant) } @@ -253,6 +406,28 @@ const attach = ( } loaded[name] = byKey } + const counts: Record> = {} + for (const [name, spec] of counted) { + const relation = yield* relationOf(relations, key, name) + const target = tables[relation.target]! + const where = spec === true ? undefined : spec!.where + const keys = distinct(rows.map((row) => row[relation.from])) + const byKey = new Map() + if (keys.length > 0) { + // `SELECT , count(*) … WHERE IN () GROUP BY `: the + // same key match and tenant pin as a loaded level. + const match = keyMatch(parent, target, relation, rows, keys) + let query: CHQuery = from(target as Table) + .select(($: any) => ({ key: $[relation.to], n: countAll() })) + .where(($: any) => [...match($), ...(where?.($, relatedOf(tables, relations, relation.target, $)) ?? [])]) + .groupBy("key" as never) + if (crossTenant) query = query.crossTenant() + for (const row of (yield* db.run(query as any)) as ReadonlyArray<{ key: unknown; n: number }>) { + byKey.set(row.key, row.n) + } + } + counts[name] = byKey + } return rows.map((row) => { const out: Row = { ...row } for (const [name] of names) { @@ -260,6 +435,11 @@ const attach = ( const group = loaded[name]!.get(row[relation.from]) ?? [] out[name] = relation.kind === "many" ? group : (group[0] ?? null) } + if (counted.length > 0) { + out._count = Object.fromEntries( + counted.map(([name]) => [name, counts[name]!.get(row[relations[key]![name]!.from]) ?? 0]), + ) + } return out }) }) @@ -271,8 +451,15 @@ const attach = ( * one tenant is pinned to it. Levels are separate statements: run `findMany` * inside `transaction` for one snapshot across them. * + * `where` gets the level's relation filters as its second argument, and + * `_count` counts a `many` relation with one grouped statement per level. + * * ```ts - * Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) + * Db.findMany(schema.users, { + * where: ($, r) => [$.orgId.eq(orgId), r.posts.some()], + * with: { posts: { with: { author: true } } }, + * _count: { posts: true }, + * }) * ``` */ export const findMany: { @@ -282,12 +469,13 @@ export const findMany: { >( table: RelationalTable, options: O, - ): Effect.Effect>>, FindError, Database> + ): Effect.Effect, CountIn>>, FindError, Database> } = (table: RelationalTable, options?: FindManyOptions) => Effect.gen(function* () { const level = (options ?? {}) as Level & { readonly crossTenant?: boolean } const crossTenant = level.crossTenant === true + const relations = table.relations as Relations const db = yield* Database - const rows = (yield* db.run(levelQuery(table.table, level, () => [], crossTenant) as any)) as ReadonlyArray - return yield* attach(table.tables, table.relations as any, table.key, rows, level, crossTenant) + const rows = (yield* db.run(levelQuery(table.tables, relations, table.key, level, () => [], crossTenant) as any)) as ReadonlyArray + return yield* attach(table.tables, relations, table.key, rows, level, crossTenant) }) as any From 9af3aabc27b3bd6516fcf7f8ef929ae9c8238abf Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:39:24 +0200 Subject: [PATCH 5/9] test(database): relation filters and counts on PGlite and ClickHouse --- src/database/relation-filters.test-d.ts | 80 ++++++ src/database/relation-filters.test.ts | 330 ++++++++++++++++++++++ tests/relation-filters.clickhouse.test.ts | 149 ++++++++++ 3 files changed, 559 insertions(+) create mode 100644 src/database/relation-filters.test-d.ts create mode 100644 src/database/relation-filters.test.ts create mode 100644 tests/relation-filters.clickhouse.test.ts diff --git a/src/database/relation-filters.test-d.ts b/src/database/relation-filters.test-d.ts new file mode 100644 index 0000000..68c84f9 --- /dev/null +++ b/src/database/relation-filters.test-d.ts @@ -0,0 +1,80 @@ +// Type-level tests: relation filters and counts follow the relation schema, and +// `_count` adds a typed `_count` to the rows of its level. + +import type { Effect } from "effect" +import { expectTypeOf } from "expect-type" +import * as PG from "../postgres" +import * as Db from "../database" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.int4, title: PG.text }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +type FindError = Db.DatabaseError | PG.QueryBuilderError | PG.CompiledQueryDecodeError + +// `_count` adds `_count` with one number per counted relation, at any level. +expectTypeOf( + Db.findMany(schema.users, { + _count: { posts: true }, + with: { posts: { with: { author: { _count: { posts: { where: ($) => [$.title.eq("a")] } } } } } }, + }), +).toEqualTypeOf< + Effect.Effect< + ReadonlyArray<{ + readonly id: number + readonly name: string + readonly _count: { readonly posts: number } + readonly posts: ReadonlyArray<{ + readonly id: number + readonly authorId: number + readonly title: string + readonly author: { + readonly id: number + readonly name: string + readonly _count: { readonly posts: number } + } | null + }> + }>, + FindError, + Db.Database + > +>() + +// `_count` takes `many` relations only, and only relation names. +// @ts-expect-error `author` is a `one` relation +Db.findMany(schema.posts, { _count: { author: true } }) +// @ts-expect-error no relation named `comments` +Db.findMany(schema.users, { _count: { comments: true } }) + +// The filters see the target's columns, and the next level's relations. +Db.findMany(schema.users, { + where: (_, r) => [ + r.posts.some(($) => [$.title.eq("a")]), + r.posts.every(($, r) => [r.author.none(($) => [$.name.eq("b")])]), + // @ts-expect-error `name` is not a column of posts + r.posts.none(($) => [$.name.eq("a")]), + ], +}) +// @ts-expect-error no relation named `comments` +Db.findMany(schema.users, { where: (_, r) => [r.comments.some()] }) +// @ts-expect-error `every` needs its condition +Db.findMany(schema.users, { where: (_, r) => [r.posts.every()] }) + +// On the builder: a count is a number, and params inside a filter reach the query. +PG.from(Users).select(($) => { + expectTypeOf(Db.related(schema.users, $).posts.count()).toEqualTypeOf>() + return { id: $.id } +}) +const withParam = PG.from(Users) + .select(($) => ({ id: $.id })) + .where(($) => [Db.related(schema.users, $).posts.some(($) => [$.title.eq(PG.param.string("title"))])]) +expectTypeOf(PG.compileUnsafe(withParam, { title: "a" }).sql).toEqualTypeOf() +// @ts-expect-error the `title` param is required +PG.compileUnsafe(withParam, {}) diff --git a/src/database/relation-filters.test.ts b/src/database/relation-filters.test.ts new file mode 100644 index 0000000..925404a --- /dev/null +++ b/src/database/relation-filters.test.ts @@ -0,0 +1,330 @@ +// Relation filters (`some`, `none`, `every`), relation counts, and `_count` in +// `findMany`, on PGlite. The filters are correlated EXISTS subqueries; the +// tenant tests check that the subquery stays inside the outer query's tenant. + +import { PgliteClient } from "@effect/sql-pglite" +import { assert, describe, expect, it as plain, layer } from "@effect/vitest" +import { Cause, Effect, Exit, Layer } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as PG from "../postgres" +import { postgresDialect } from "../pg/dialect" +import * as Db from "../database" + +const statements: Array = [] + +const live = (requireTenantScope: boolean) => + Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + requireTenantScope, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), + ).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { + id: PG.int4, + authorId: PG.column(PG.int4, { name: "author_id" }), + title: PG.text, + published: PG.bool, + }, + primaryKey: ["id"], +}) +const Comments = PG.table("comments", { + columns: { id: PG.int4, postId: PG.column(PG.int4, { name: "post_id" }), body: PG.text }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts, comments: Comments }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { + author: r.one("users", { from: "authorId", to: "id" }), + comments: r.many("comments", { from: "id", to: "postId" }), + }, +})) + +const setup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute( + Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL, published bool NOT NULL)`, + ) + yield* Db.execute(Db.sql`CREATE TABLE comments (id int4 PRIMARY KEY, post_id int4 NOT NULL, body text NOT NULL)`) + // ada: one published and one draft post; bob: one published post; cy: none. + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob'), (3, 'cy')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'a1', true), (11, 1, 'a2', false), (12, 2, 'b1', true)`) + yield* Db.execute(Db.sql`INSERT INTO comments VALUES (100, 10, 'nice'), (101, 12, 'ok')`) +}) + +const names = (rows: ReadonlyArray<{ readonly name: string }>) => rows.map((row) => row.name) + +layer(live(false), { excludeTestServices: true })("relation filters on PGlite", (it) => { + it.effect("some: at least one related row matches", () => + Effect.gen(function* () { + yield* setup + const any = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.some()], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(any)).toEqual(["ada", "bob"]) + const drafts = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.some(($) => [$.published.eq(false)])], + }) + expect(names(drafts)).toEqual(["ada"]) + }), + ) + + it.effect("none: no related row matches", () => + Effect.gen(function* () { + const noPosts = yield* Db.findMany(schema.users, { where: (_, r) => [r.posts.none()] }) + expect(names(noPosts)).toEqual(["cy"]) + const noDrafts = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.none(($) => [$.published.eq(false)])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(noDrafts)).toEqual(["bob", "cy"]) + }), + ) + + it.effect("every: all related rows match, and a row with none passes", () => + Effect.gen(function* () { + const allPublished = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.every(($) => [$.published.eq(true)])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(allPublished)).toEqual(["bob", "cy"]) + }), + ) + + it.effect("filters nest, through a one relation too", () => + Effect.gen(function* () { + // Users with a post that has a comment. + const commented = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.some((_, r) => [r.comments.some()])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(commented)).toEqual(["ada", "bob"]) + // Posts whose author has a draft. + const posts = yield* Db.findMany(schema.posts, { + where: (_, r) => [r.author.some((_, r) => [r.posts.some(($) => [$.published.eq(false)])])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(posts.map((post) => post.id)).toEqual([10, 11]) + }), + ) + + it.effect("a filter applies on a nested level of with", () => + Effect.gen(function* () { + const users = yield* Db.findMany(schema.users, { + where: ($) => [$.id.eq(1)], + with: { posts: { where: (_, r) => [r.comments.none()] } }, + }) + expect(users[0]!.posts.map((post) => post.id)).toEqual([11]) + }), + ) + + it.effect("on the builder: related() filters a where and counts in a select", () => + Effect.gen(function* () { + const query = PG.from(Users) + .select(($) => ({ + name: $.name, + posts: Db.related(schema.users, $).posts.count(), + published: Db.related(schema.users, $).posts.count(($) => [$.published.eq(true)]), + })) + .where(($) => [Db.related(schema.users, $).posts.none(($) => [$.title.eq("b1")])]) + .orderBy(["name", "asc"]) + const rows = yield* Db.run(query) + expect(rows).toEqual([ + { name: "ada", posts: 2, published: 1 }, + { name: "cy", posts: 0, published: 0 }, + ]) + }), + ) + + it.effect("a relation count works in a where", () => + Effect.gen(function* () { + const query = PG.from(Users) + .select(($) => ({ name: $.name })) + .where(($) => [Db.related(schema.users, $).posts.count().gte(2)]) + expect(yield* Db.run(query)).toEqual([{ name: "ada" }]) + }), + ) + + it.effect("_count: one grouped statement per counted relation, 0 for none", () => + Effect.gen(function* () { + statements.length = 0 + const users = yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + _count: { posts: true }, + }) + expect(users.map((user) => [user.name, user._count.posts])).toEqual([ + ["ada", 2], + ["bob", 1], + ["cy", 0], + ]) + expect(statements).toHaveLength(2) + expect(statements[1]).toMatch(/count\(\*\)/) + expect(statements[1]).toMatch(/GROUP BY/) + expect(statements[1]).toMatch(/IN \(/) + }), + ) + + it.effect("_count takes a where, and works on a nested level", () => + Effect.gen(function* () { + const users = yield* Db.findMany(schema.users, { + where: ($) => [$.id.eq(1)], + _count: { posts: { where: ($) => [$.published.eq(true)] } }, + with: { posts: { orderBy: ($) => [[$.id, "asc"]], _count: { comments: true } } }, + }) + expect(users).toEqual([ + { + id: 1, + name: "ada", + _count: { posts: 1 }, + posts: [ + { id: 10, authorId: 1, title: "a1", published: true, _count: { comments: 1 } }, + { id: 11, authorId: 1, title: "a2", published: false, _count: { comments: 0 } }, + ], + }, + ]) + }), + ) +}) + +describe("relation filters and the subquery alias", () => { + const Nodes = PG.table("nodes", { + columns: { id: PG.int4, parentId: PG.column(PG.nullable(PG.int4), { name: "parent_id" }) }, + primaryKey: ["id"], + }) + const tree = Db.defineRelations({ nodes: Nodes }, (r) => ({ + nodes: { children: r.many("nodes", { from: "id", to: "parentId" }) }, + })) + + plain("a table related to itself reads the subquery under its own alias", () => { + const compiled = PG.compileUnsafe( + PG.from(Nodes) + .select(($) => ({ id: $.id })) + .where(($) => [Db.related(tree.nodes, $).children.some()]), + {}, + ) + expect(compiled.sql).toMatch(/FROM "nodes" AS "nodes__children"\s+WHERE "nodes__children"."parent_id" = "nodes"."id"/) + }) +}) + +// Tenant scope: the subquery is pinned to the outer row's tenant. +const Orgs = PG.table("orgs", { columns: { id: PG.text }, primaryKey: ["id"] }) +const Units = PG.table("units", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const Locks = PG.table("locks", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }), unitId: PG.column(PG.int4, { name: "unit_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const Gateways = PG.table("gateways", { + columns: { id: PG.int4, lockId: PG.column(PG.int4, { name: "lock_id" }) }, + primaryKey: ["id"], +}) +const tenancy = Db.defineRelations({ orgs: Orgs, units: Units, locks: Locks, gateways: Gateways }, (r) => ({ + orgs: { units: r.many("units", { from: "id", to: "orgId" }) }, + units: { locks: r.many("locks", { from: "id", to: "unitId" }) }, + locks: { gateways: r.many("gateways", { from: "id", to: "lockId" }) }, +})) + +describe("relation filters keep the tenant scope", () => { + const scope = (query: PG.CHQuery) => PG.compileUnsafe(query, {}).tenantScope + + plain("a filter under a scoped outer query is single-tenant, with the tenant equality in the subquery", () => { + const query = PG.from(Units) + .select(($) => ({ id: $.id })) + .where(($) => [$.orgId.eq("a"), Db.related(tenancy.units, $).locks.some()]) + const compiled = PG.compileUnsafe(query, {}) + expect(compiled.tenantScope).toBe("single-tenant") + expect(compiled.sql).toContain(`"units__locks"."org_id" = "units"."org_id"`) + }) + + plain("nested filters and counts stay single-tenant", () => { + expect( + scope( + PG.from(Units) + .select(($) => ({ id: $.id, n: Db.related(tenancy.units, $).locks.count() })) + .where(($) => [$.orgId.eq("a"), Db.related(tenancy.units, $).locks.some((_, r) => [r.gateways.none()])]), + ), + ).toBe("single-tenant") + }) + + plain("a filter does not scope an unscoped outer query", () => { + expect( + scope( + PG.from(Units) + .select(($) => ({ id: $.id })) + .where(($) => [Db.related(tenancy.units, $).locks.some(($) => [$.orgId.eq("a")])]), + ), + ).toBe("cross-tenant") + }) + + plain("a filter from an untenanted table into a tenanted one needs its own tenant equality", () => { + const orgs = (where: (r: Db.Related) => PG.Condition) => + scope(PG.from(Orgs).select(($) => ({ id: $.id })).where(($) => [where(Db.related(tenancy.orgs, $))])) + expect(orgs((r) => r.units.some())).toBe("cross-tenant") + expect(orgs((r) => r.units.some(($) => [$.orgId.eq("a")]))).toBe("single-tenant") + }) + + plain("a filter whose own tenant equality names another tenant is cross-tenant", () => { + expect( + scope( + PG.from(Units) + .select(($) => ({ id: $.id })) + .where(($) => [$.orgId.eq("a"), Db.related(tenancy.units, $).locks.some(($) => [$.orgId.eq("b")])]), + ), + ).toBe("cross-tenant") + }) +}) + +layer(live(true), { excludeTestServices: true })("relation filters under requireTenantScope", (it) => { + const tenantSetup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE orgs (id text PRIMARY KEY)`) + yield* Db.execute(Db.sql`CREATE TABLE units (id int4 PRIMARY KEY, org_id text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE locks (id int4 PRIMARY KEY, org_id text NOT NULL, unit_id int4 NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE gateways (id int4 PRIMARY KEY, lock_id int4 NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO orgs VALUES ('a'), ('b')`) + yield* Db.execute(Db.sql`INSERT INTO units VALUES (1, 'a'), (2, 'a'), (3, 'b')`) + // Lock 21 points at unit 2 but belongs to org b: bad data the tenant equality must not count. + yield* Db.execute(Db.sql`INSERT INTO locks VALUES (10, 'a', 1), (11, 'a', 1), (21, 'b', 2), (30, 'b', 3)`) + }) + + it.effect("filters and _count run scoped, and a row of another tenant does not match", () => + Effect.gen(function* () { + yield* tenantSetup + const units = yield* Db.findMany(tenancy.units, { + where: ($, r) => [$.orgId.eq("a"), r.locks.none()], + }) + expect(units.map((unit) => unit.id)).toEqual([2]) + const counted = yield* Db.findMany(tenancy.units, { + where: ($) => [$.orgId.eq("a")], + orderBy: ($) => [[$.id, "asc"]], + _count: { locks: true }, + }) + expect(counted.map((unit) => [unit.id, unit._count.locks])).toEqual([ + [1, 2], + [2, 0], + ]) + }), + ) + + it.effect("an unscoped filter is refused", () => + Effect.gen(function* () { + const exit = yield* Effect.exit( + Db.findMany(tenancy.orgs, { where: ($, r) => [$.id.eq("a"), r.units.some()] }), + ) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + }), + ) +}) diff --git a/tests/relation-filters.clickhouse.test.ts b/tests/relation-filters.clickhouse.test.ts new file mode 100644 index 0000000..6d4cb01 --- /dev/null +++ b/tests/relation-filters.clickhouse.test.ts @@ -0,0 +1,149 @@ +// Relation filters and counts on a live ClickHouse. `some` / `none` / `every` +// and `count()` are correlated subqueries, which ClickHouse runs since 25.x +// (`allow_experimental_correlated_subqueries`, on by default in 26.8). +// `_count` in `findMany` is a grouped `IN` query, which needs no correlation. + +import { ClickhouseClient } from "@effect/sql-clickhouse" +import { Effect } from "effect" +import { describe, expect, it } from "vitest" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as Db from "@maple-dev/effect-orm/database" +import { endpoint } from "./clickhouse-support" + +const user = process.env.EFFECT_ORM_CLICKHOUSE_USER ?? "default" +const password = process.env.EFFECT_ORM_CLICKHOUSE_PASSWORD ?? "" + +const Services = CH.table("services", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, Id: CH.uint32, Name: CH.string }, +}) +const Events = CH.table("events", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, Id: CH.uint32, ServiceId: CH.uint32, Level: CH.string }, +}) +const Notes = CH.table("notes", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, EventId: CH.uint32 }, +}) +const schema = Db.defineRelations({ services: Services, events: Events, notes: Notes }, (r) => ({ + services: { events: r.many("events", { from: "Id", to: "ServiceId" }) }, + events: { notes: r.many("notes", { from: "Id", to: "EventId" }) }, +})) + +const withDatabase = (body: (db: Db.Database["Service"]) => Effect.Effect) => + Effect.runPromise( + Effect.gen(function* () { + const database = `eo_relation_filters_${Date.now()}_${Math.floor(Math.random() * 1e6)}` + const admin = yield* ClickhouseClient.ClickhouseClient + yield* admin.asCommand(admin.unsafe(`CREATE DATABASE ${database}`)) + return yield* Effect.gen(function* () { + const client = yield* ClickhouseClient.ClickhouseClient + const db = Db.fromSqlClient(client, { + dialect: CH.clickhouseDialect, + command: client.asCommand, + requireTenantScope: true, + }) + yield* db.execute(Db.sql`CREATE TABLE services (OrgId String, Id UInt32, Name String) ENGINE = MergeTree ORDER BY Id`) + yield* db.execute( + Db.sql`CREATE TABLE events (OrgId String, Id UInt32, ServiceId UInt32, Level String) ENGINE = MergeTree ORDER BY Id`, + ) + yield* db.execute(Db.sql`CREATE TABLE notes (OrgId String, EventId UInt32) ENGINE = MergeTree ORDER BY EventId`) + // api: an error and an info event; web: one info event; db: none. + // Event 3 points at api but belongs to org b: the tenant equality must skip it. + yield* db.execute(Db.sql`INSERT INTO services VALUES ('a', 1, 'api'), ('a', 2, 'web'), ('a', 3, 'db'), ('b', 4, 'other')`) + yield* db.execute( + Db.sql`INSERT INTO events VALUES ('a', 1, 1, 'error'), ('a', 2, 1, 'info'), ('a', 4, 2, 'info'), ('b', 3, 1, 'error')`, + ) + yield* db.execute(Db.sql`INSERT INTO notes VALUES ('a', 1)`) + return yield* body(db).pipe(Effect.provideService(Db.Database, db)) + }).pipe( + Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database })), + Effect.ensuring(Effect.orDie(admin.asCommand(admin.unsafe(`DROP DATABASE IF EXISTS ${database} SYNC`)))), + ) + }).pipe(Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database: "default" }))), + ) + +const names = (rows: ReadonlyArray<{ readonly Name: string }>) => rows.map((row) => row.Name) + +describe.skipIf(!endpoint)("relation filters on live ClickHouse", () => { + it("some, none and every run as correlated subqueries, scoped to the tenant", async () => { + const result = await withDatabase(() => + Effect.gen(function* () { + const find = (where: Db.RelationWhere) => + Db.findMany(schema.services, { + where: ($, r) => [$.OrgId.eq("a"), ...where($, r)], + orderBy: ($) => [[$.Id, "asc"]], + }).pipe(Effect.map(names)) + return { + some: yield* find((_, r) => [r.events.some()]), + errors: yield* find((_, r) => [r.events.some(($) => [$.Level.eq("error")])]), + none: yield* find((_, r) => [r.events.none()]), + every: yield* find((_, r) => [r.events.every(($) => [$.Level.eq("info")])]), + nested: yield* find((_, r) => [r.events.some((_, r) => [r.notes.some()])]), + } + }), + ) + expect(result).toEqual({ + some: ["api", "web"], + errors: ["api"], + none: ["db"], + every: ["web", "db"], + nested: ["api"], + }) + }) + + // ClickHouse 26.8 runs a correlated subquery in SELECT next to a WHERE only + // when the outer columns it correlates on are selected too ("can't find + // correlated column … in current header", NOT_IMPLEMENTED). So this selects + // the key (`Id`) and the tenant column (`OrgId`) beside the counts. + it("count() in a select and a where, 0 when nothing matches", async () => { + const rows = await withDatabase(() => + Db.run( + CH.from(Services) + .select(($) => ({ + org: $.OrgId, + id: $.Id, + name: $.Name, + events: Db.related(schema.services, $).events.count(), + errors: Db.related(schema.services, $).events.count(($) => [$.Level.eq("error")]), + })) + .where(($) => [$.OrgId.eq("a"), Db.related(schema.services, $).events.count().lt(2)]) + .orderBy(["name", "asc"]), + ), + ) + expect(rows).toEqual([ + { org: "a", id: 3, name: "db", events: 0, errors: 0 }, + { org: "a", id: 2, name: "web", events: 1, errors: 0 }, + ]) + }) + + it("count() in a where needs no selected key", async () => { + const rows = await withDatabase(() => + Db.run( + CH.from(Services) + .select(($) => ({ name: $.Name })) + .where(($) => [$.OrgId.eq("a"), Db.related(schema.services, $).events.count().gte(2)]), + ), + ) + expect(rows).toEqual([{ name: "api" }]) + }) + + it("_count in findMany is one grouped statement per level", async () => { + const rows = await withDatabase(() => + Db.findMany(schema.services, { + where: ($) => [$.OrgId.eq("a")], + orderBy: ($) => [[$.Id, "asc"]], + _count: { events: true }, + with: { events: { orderBy: ($) => [[$.Id, "asc"]], _count: { notes: true } } }, + }), + ) + expect(rows.map((row) => [row.Name, row._count.events, row.events.map((event) => event._count.notes)])).toEqual([ + ["api", 2, [1, 0]], + ["web", 1, [0]], + ["db", 0, []], + ]) + }) +}) From c1717c9a438741492f0bd481e7c39ba5e5673b83 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:41:36 +0200 Subject: [PATCH 6/9] fix(database): refuse relation filters on an unqualified accessor --- src/database/relation-filters.test.ts | 5 +++++ src/database/relations.ts | 25 +++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/database/relation-filters.test.ts b/src/database/relation-filters.test.ts index 925404a..0882308 100644 --- a/src/database/relation-filters.test.ts +++ b/src/database/relation-filters.test.ts @@ -196,6 +196,11 @@ layer(live(false), { excludeTestServices: true })("relation filters on PGlite", }) describe("relation filters and the subquery alias", () => { + plain("a filter on an update's or a delete's unqualified $ is refused, not correlated to itself", () => { + const query = PG.deleteFrom(Users).where(($) => [Db.related(schema.users, $ as any).posts.none()]) + expect(() => PG.compileUnsafe(query, {})).toThrow(/an update or delete has no alias/) + }) + const Nodes = PG.table("nodes", { columns: { id: PG.int4, parentId: PG.column(PG.nullable(PG.int4), { name: "parent_id" }) }, primaryKey: ["id"], diff --git a/src/database/relations.ts b/src/database/relations.ts index f10d5b8..ccdb1ec 100644 --- a/src/database/relations.ts +++ b/src/database/relations.ts @@ -8,7 +8,7 @@ import { Effect } from "effect" import type { CompiledQueryDecodeError } from "../ch/compile" -import type { QueryBuilderError } from "../ch/errors" +import { QueryBuilderDefect, type QueryBuilderError } from "../ch/errors" import { and, lit, makeCond, makeExpr, not, type Condition, type Expr, type ParamsIn } from "../ch/expr" import { builtins } from "../ch/functions/builtin" import { from, type ColumnAccessor, type CHQuery } from "../ch/query" @@ -17,7 +17,8 @@ import type { SelectRowOf, Table } from "../ch/table" import { markCorrelatedTenant, tenantColumnOf } from "../ch/tenant" import { custom, type ColumnDefs, type InferTS } from "../ch/types" import { PgNumber } from "../pg/types" -import { raw } from "../sql/sql-fragment" +import { fail } from "../ch/failure" +import { known, raw } from "../sql/sql-fragment" import { Database } from "./database" import type { DatabaseError } from "./errors" @@ -182,6 +183,25 @@ const countType = custom("Int64", PgNumber) const countAll = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), countType.schema) const always = (): Condition => makeCond(raw("1 = 1")) +/** + * The filters of a relation on an accessor whose columns are not qualified, as + * an update's or a delete's. The correlation would name an inner column, so + * each filter fails the compile instead. + */ +const unqualified = (key: string, name: string): RelationFilter => { + const failure = (placeholder: string) => + known(() => + fail( + new QueryBuilderDefect({ + message: `related(${key}).${name}: $ must be the accessor of a select over ${key}; an update or delete has no alias to correlate on`, + }), + placeholder, + ), + ) + const condition = () => makeCond(failure("FALSE")) + return { some: condition, none: condition, every: condition, count: () => makeExpr(failure("0"), countType.schema) as any } +} + const relatedOf = ( tables: RelationTables, relations: Record | undefined>, @@ -196,6 +216,7 @@ const relatedOf = ( // `$` is qualified (`alias.column`): the subquery's alias extends it, so // it never hides the outer source, even for a table related to itself. const outerName: string = outerFrom.columnName + if (!outerName.endsWith(`.${relation.from}`)) return [name, unqualified(key, name)] const alias = `${outerName.slice(0, outerName.length - relation.from.length - 1)}__${name}` const outerTenant = parent.tenantColumn !== undefined && target.tenantColumn !== undefined ? $[parent.tenantColumn] : undefined From 8be9e667c84743639e49c5accd91ca4448478e86 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:41:36 +0200 Subject: [PATCH 7/9] docs: relation filters, relation counts and _count --- .changeset/relation-filters.md | 5 ++ design/relations.md | 59 +++++++++++++++++- docs/reference.md | 1 + docs/relations.md | 109 ++++++++++++++++++++++++++++++++- 4 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 .changeset/relation-filters.md diff --git a/.changeset/relation-filters.md b/.changeset/relation-filters.md new file mode 100644 index 0000000..cec33db --- /dev/null +++ b/.changeset/relation-filters.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +Relation filters and counts in `/database`, as Prisma's `some` / `none` / `every` and `_count`. `findMany`'s `where` gets the relations of its level as a second argument: `where: ($, r) => [r.posts.some(($) => [$.published.eq(false)])]`. `Db.related(schema.users, $)` gives the same filters on a builder query, and `r.posts.count(where?)`, an `Expr` for a `select` or a `where`. The filters are correlated `EXISTS` / `NOT EXISTS` subqueries. `_count: { posts: true }` in `findMany` adds `_count: { posts: number }` to each row, with one grouped statement per level. When both tables declare a tenant column, the subquery matches the outer row's tenant, and a filter under a scoped query stays `"single-tenant"`. See docs/relations.md. diff --git a/design/relations.md b/design/relations.md index a7f5377..a7a51a0 100644 --- a/design/relations.md +++ b/design/relations.md @@ -2,7 +2,8 @@ Status: first slice built (issue #79; `docs/relations.md`). `one` and `many` relations over single-column keys, `with` to any depth, `where` and `orderBy` per level, `limit` and `offset` on -the top level. Section 5 lists what is not built. +the top level. Relation filters and counts (issue #80) are in section 6. Section 5 lists what is +not built. ## 1. Goal @@ -103,6 +104,60 @@ grandchild reached through a shared table. | `limit` per parent on a nested level | Needs `row_number()` over a partition, or a lateral join | | Composite keys | The loader would match tuples; no issue asks for it yet | | `findFirst` | `findMany` with `limit: 1`; cardinality helpers are #69 | -| `_count`, `some` / `every` / `none` | Issue #80 | +| `_count` of a `one` relation | `r.author.some()` says the same; no call site needs a 0/1 count | +| Relation filters in `update` / `delete` | Their accessor is not qualified, so there is no alias to correlate on; refused | +| `orderBy` a relation count | Select `count()` on a builder query and order by its alias | | Params in nested `where` | `findMany` takes no params; values go in as literals | | SQLite, MySQL, D1 | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | + +## 6. Relation filters and counts + +```ts +Db.findMany(schema.users, { + where: ($, r) => [$.orgId.eq(orgId), r.posts.some(($) => [$.published.eq(false)])], + _count: { posts: true }, +}) +PG.from(Users).select(($) => ({ id: $.id, posts: Db.related(schema.users, $).posts.count() })) +``` + +- **Filters are correlated subqueries.** `some` is `EXISTS`, `none` is `NOT EXISTS`, `every` is + `NOT EXISTS (… AND NOT (where))`, as Prisma compiles them. A filter is a condition inside one + statement, so it cannot be a separate level: the outer rows it filters do not exist yet. The + subquery reads the target under the alias `__`, which differs from every + enclosing alias, so a self-relation needs nothing special. +- **`Db.related(schema.table, $)` is the one entry point.** It builds the filters from the `$` of + any select over the table, so the builder and `findMany` share one implementation. `findMany` + passes it as the second argument of every `where`. +- **`_count` in `findMany` is a grouped level, not a subquery.** `SELECT , count(*) … + WHERE IN () GROUP BY `, with the same tenant pin as a loaded level. This keeps + #79's model: no correlation, so it runs on every dialect and every ClickHouse version, and the + count decodes through one codec. `r.posts.count()` is the correlated form, for builder queries, + wrapped in `coalesce(…, 0)` because ClickHouse returns NULL for a correlated count over no rows. + +### Tenant scope inside the subquery + +A subquery's scope counts as one more source of the outer query (docs/tenant-scoping.md), so a +subquery that pins no tenant made every filter `"cross-tenant"`. Writing the tenant value into +the subquery is not possible: the filter is built before the outer `where` is known, and the +value can be a param. + +So when both tables declare a tenant column, the filter writes +` = ` into the subquery and marks the subquery with the outer column +(`markCorrelatedTenant`, internal). Compiled as a subquery of the outer query, the marked query's +main source starts bound to a token for that outer column. If the subquery is single-tenant on +that token, the outer compile counts it as a source on the outer tenant column, which the outer +`where` binds like any other. Nested filters chain: the inner filter's token resolves against the +middle subquery's own main column. + +This is sound because only the filters set the mark, and they always write the equality the mark +claims. A subquery whose own `where` pins a literal tenant is bound to that literal instead, and +the outer query must pin the same one. A marked query compiled alone ignores the mark. The write +paths (insert, update, delete) do not read the mark, so a marked subquery there stays +`"cross-tenant"`. + +### ClickHouse + +Correlated subqueries run on ClickHouse 25 and later. On 26.8 (CI): `EXISTS` / `NOT EXISTS`, +nested, and a correlated count in `WHERE` work. A correlated scalar subquery in `SELECT` next to a +`WHERE` fails with `NOT_IMPLEMENTED` unless the correlated outer columns are also selected. The +docs say to select them, or to use `_count`, which needs no correlation. diff --git a/docs/reference.md b/docs/reference.md index f34bc3c..64f5535 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -514,6 +514,7 @@ placeholders, double-quoted identifiers. | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | | `defineRelations`, `findMany`, `Relation`, `RelationBuilder`, `RelationsConfig`, `RelationTables`, `RelationalSchema`, `RelationalTable`, `RelationOptions`, `FindManyOptions`, `FindManyRow`, `WithOf`, `KeyColumn` — relational reads, one statement per level ([Relational reads](./relations.md)) | `/database` | +| `related`, `Related`, `RelationFilter`, `RelationWhere`, `CountOf` — relation filters `some` / `none` / `every`, relation counts, and `_count` in `findMany` ([Filter by relations](./relations.md#filter-by-relations)) | `/database` | | `withQuerySummary`, `QuerySummary` — the `db.query.summary` of the statements an effect runs, for spans | `/database` | | `ClickhouseHttpClient`, `layer`, `make`, `ClickhouseHttpClientConfig` — a ClickHouse `SqlClient` over `fetch`, for Workers | `/clickhouse-http` | | `defineConfig`, `generate`, `check`, `pull`, `loadSchema`, `readMigrations`, `analyze`, `KitError` | `/kit` | diff --git a/docs/relations.md b/docs/relations.md index 173046e..941a3ba 100644 --- a/docs/relations.md +++ b/docs/relations.md @@ -68,13 +68,15 @@ _(Backed by `src/database/relations.test-d.ts`, and `tests/relations.clickhouse. ## Read with `findMany` `findMany(schema.table, options?)` returns every column of the table, plus each relation that -`with` names. `with` takes `true`, or the options of that level: +`with` names, plus `_count` when the level counts relations. `with` takes `true`, or the options +of that level: | Option | Levels | Meaning | | --- | --- | --- | | `where` | All | More conditions, ANDed with the relation's key match | | `orderBy` | All | The order of the rows; nested rows keep it within each parent | | `with` | All | Relations to load under each row of this level | +| `_count` | All | `many` relations to count under each row of this level | | `limit`, `offset` | Top only | Paging of the top level | | `crossTenant` | Top only | Declare every level `.crossTenant()` | @@ -117,13 +119,116 @@ cross-tenant and runs. _(Backed by `src/database/relations.test.ts`.)_ +## Filter by relations + +A `where` gets a second argument, `r`, with the relations of its table. Each relation has three +filters, as Prisma's `some`, `none` and `every`: + +| Filter | True when | SQL | +| --- | --- | --- | +| `r.posts.some(where?)` | At least one related row matches (with no `where`: there is one) | `EXISTS (…)` | +| `r.posts.none(where?)` | No related row matches (with no `where`: there is none) | `NOT EXISTS (…)` | +| `r.posts.every(where)` | Every related row matches; also true when there is none | `NOT EXISTS (… AND NOT (where))` | + +```ts title="relations-filters.ts" +import { Effect } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), published: PG.bool }, + primaryKey: ["id"], +}) +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +// Users with a draft, and users whose posts are all published. +export const withDrafts = Db.findMany(schema.users, { where: (_, r) => [r.posts.some(($) => [$.published.eq(false)])] }) +export const allPublished = Db.findMany(schema.users, { where: (_, r) => [r.posts.every(($) => [$.published.eq(true)])] }) + +// The same filters on a builder query: `Db.related` takes the query's `$`. +export const query = PG.from(Users) + .select(($) => ({ id: $.id, posts: Db.related(schema.users, $).posts.count() })) + .where(($) => [Db.related(schema.users, $).posts.none()]) +export const rows: Effect.Effect, unknown, Db.Database> = + Db.run(query) +``` + +Each filter is one correlated subquery in the statement that holds it. It reads the target +table under its own alias, `__`, so a table related to itself works. The +`where` of a filter gets its own `r`, so filters nest: `r.posts.some((_, r) => [r.comments.some()])`. +A `one` relation has the same filters: `some` is "the related row exists and matches", `none` is +"it does not". + +`every` negates the condition, so a related row where the condition is NULL passes it. Use a +condition that cannot be NULL, or say what NULL means with `isNull` / `isNotNull`. + +_(Backed by `src/database/relation-filters.test.ts`, and `tests/relation-filters.clickhouse.test.ts` on ClickHouse.)_ + +## Count relations + +Two ways, for two shapes of read: + +- **`_count` in `findMany`**, as Prisma's `_count`. `_count: { posts: true }` adds + `_count: { posts: number }` to each row of that level. `_count: { posts: { where } }` counts + only the rows that match. Only a `many` relation can be counted. It runs one statement per + counted relation, the same way as a level of `with`: + `SELECT , count(*) FROM WHERE IN () [AND ] GROUP BY `. + A row with no related rows gets `0`. +- **`r.posts.count(where?)` on a query**, an `Expr`: a correlated + `coalesce((SELECT count(*) …), 0)`. Use it in a builder `select`, or in a `where` such as + `Db.related(schema.users, $).posts.count().gte(2)`. + +_(Backed by `src/database/relation-filters.test.ts`.)_ + +## Tenant scope in filters and counts + +When both tables declare a `tenantColumn`, a filter's subquery also gets +` = `. The compiler then gives the subquery the +tenant the outer query is bound to: a filter under `$.orgId.eq("a")` is `"single-tenant"`. The +equality also hides a related row whose tenant differs from the outer row's. + +A filter does not scope the query that holds it. Without a tenant filter of its own, the outer +query stays `"cross-tenant"`. A filter from an untenanted table into a tenanted one has no outer +tenant to match, so it is `"cross-tenant"` too, unless its own `where` pins the tenant +(`r.units.some(($) => [$.orgId.eq("a")])`). A filter whose `where` pins a different tenant from the +outer query makes the query `"cross-tenant"`. + +`_count` uses the same pin as a level of `with` (see above). + +_(Backed by `src/database/relation-filters.test.ts`.)_ + +## On ClickHouse + +ClickHouse 25 and later run correlated subqueries (`allow_experimental_correlated_subqueries`, +on by default in 26.8). On 26.8: + +- `some`, `none`, `every`, and filters nested in filters work. +- `count()` in a `where` works. +- `count()` in a `select` works when the outer columns it correlates on are selected too, or the + query has no `where`. Otherwise ClickHouse refuses the query with `NOT_IMPLEMENTED` + ("can't find correlated column"). Select the relation's `from` column and the tenant column + next to the count, or use `_count` in `findMany`. +- A correlated `count()` over no rows returns NULL on ClickHouse, not 0. The `coalesce` in + `count()` turns it into 0. +- `_count` needs no correlation, so it works on every version. + +_(Backed by `tests/relation-filters.clickhouse.test.ts`.)_ + ## Not yet - Choosing columns per level (`columns: { id: true }`). Every level reads every column. - A `limit` per parent on a nested level. That needs a window function or a lateral join. - Composite keys (`from` and `to` with more than one column). - `findFirst`. Use `findMany` with `limit: 1`. -- Filters and counts over relations (`_count`, `some`, `every`, `none`). +- `_count` of a `one` relation. Test for the row with `r.author.some()`. +- Relation filters in `update` and `delete`. Their `$` has no alias to correlate on, so the + compile dies with a `QueryBuilderDefect`. Select the keys first, then write by key. +- Ordering by a relation count (`orderBy: { posts: { _count: "desc" } }`). On a builder query, + select the count and order by its alias. - `param.*` markers in a `where`: `findMany` takes no params, so write values in. See [design/relations.md](../design/relations.md) for the design and the alternatives. From f32db0aa0f355adc2bfe549e67ebac917555b31f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 2 Nov 2026 16:33:16 +0100 Subject: [PATCH 8/9] fix(database): split findMany IN lists to the dialect's bound-value limit A level with more keys than D1's 100 bindings failed to compile. Each level now sends one IN statement per batch that fits maxParameters next to its other bindings. Adds SQLite and D1 coverage and updates the docs. Co-Authored-By: Claude Opus 5.5 --- .changeset/relational-reads.md | 2 +- design/relations.md | 14 +++-- docs/relations.md | 6 +- src/database/relations.ts | 43 +++++++++---- src/sqlite/relations.test.ts | 110 +++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 20 deletions(-) create mode 100644 src/sqlite/relations.test.ts diff --git a/.changeset/relational-reads.md b/.changeset/relational-reads.md index a2858dd..b6c3d54 100644 --- a/.changeset/relational-reads.md +++ b/.changeset/relational-reads.md @@ -2,4 +2,4 @@ "@maple-dev/effect-orm": minor --- -`Db.defineRelations(tables, (r) => ...)` and `Db.findMany(schema.table, { where, orderBy, limit, with })` in `/database` read rows with their related rows, nested to any depth, as Drizzle's relational queries and Prisma's `include` do. `r.one` and `r.many` are checked against the tables. Each level of `with` is one statement, `WHERE IN (...)` over the keys of the level above, decoded through its own row codec, so a nested `timestamptz` is a `DateTime.Utc`. A level whose parent rows carry one tenant is pinned to it, and `requireTenantScope` checks every level. See docs/relations.md and design/relations.md. +`Db.defineRelations(tables, (r) => ...)` and `Db.findMany(schema.table, { where, orderBy, limit, with })` in `/database` read rows with their related rows, nested to any depth, as Drizzle's relational queries and Prisma's `include` do. `r.one` and `r.many` are checked against the tables. Each level of `with` is one statement, `WHERE IN (...)` over the keys of the level above, decoded through its own row codec, so a nested `timestamptz` is a `DateTime.Utc`. A level whose parent rows carry one tenant is pinned to it, and `requireTenantScope` checks every level. A level whose keys pass the dialect's bound-value limit (D1: 100) is split into several `IN` statements. Works on Postgres, ClickHouse, SQLite and D1. See docs/relations.md and design/relations.md. diff --git a/design/relations.md b/design/relations.md index a7f5377..8cb9799 100644 --- a/design/relations.md +++ b/design/relations.md @@ -55,10 +55,11 @@ Why: understands. A nested JSON subquery would need correlated-subquery evidence through `jsonb_agg`, which the analysis does not have yet. 3. **Every dialect.** The SQL is only `IN (...)`, so the same code runs on Postgres, on - ClickHouse, and later on SQLite / D1 and MySQL. JSON aggregation needs a different function + ClickHouse, SQLite and D1, and later on MySQL. JSON aggregation needs a different function set per dialect, and ClickHouse has no `jsonb_agg`. 4. **The statement count is fixed.** It is the depth of the `with` tree, not the row count, so - there is no N+1. + there is no N+1. The one exception is a level whose keys pass the dialect's bound-value limit + (see "Long `IN` lists" below). The costs: @@ -66,9 +67,10 @@ The costs: small. JSON aggregation is better on a slow link. - **No single snapshot.** Outside a transaction, a write can land between two levels. The docs say to use `Db.transaction({ isolationLevel: "repeatable read" })`. `findMany` does not open a - transaction itself: ClickHouse has none, and the caller may already be in one. -- **Long `IN` lists.** All keys of a level are in one list. Postgres has a limit of 65,535 bound - values. Chunking belongs with #57 (parameter limits). + transaction itself: ClickHouse and D1 have none, and the caller may already be in one. +- **Long `IN` lists.** A level's keys are split into batches that fit the dialect's + `maxParameters` next to the level's other bindings (D1 allows 100), one statement per batch. + Each parent key is in one batch, so the rows of one parent and their order do not change. JSON aggregation can come later as a second strategy behind the same API, per dialect, once the column types have JSON codecs. The result type and the tenant rule would not change. @@ -105,4 +107,4 @@ grandchild reached through a shared table. | `findFirst` | `findMany` with `limit: 1`; cardinality helpers are #69 | | `_count`, `some` / `every` / `none` | Issue #80 | | Params in nested `where` | `findMany` takes no params; values go in as literals | -| SQLite, MySQL, D1 | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | +| MySQL | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | diff --git a/docs/relations.md b/docs/relations.md index 173046e..1eb1ea3 100644 --- a/docs/relations.md +++ b/docs/relations.md @@ -93,8 +93,10 @@ The levels are separate statements. Outside a transaction, a write from another land between two of them. Run `findMany` inside [`Db.transaction`](./database.md#transactions) with `isolationLevel: "repeatable read"` for one snapshot across all levels. -All keys of a level go into one `IN` list. Postgres allows at most 65,535 bound values in one -statement, so keep the top level paged with `limit`. +A level's keys go into one `IN` list when they fit the dialect's bound-value limit (Postgres +65,535, SQLite 32,766, D1 100). Past it the level is split into several `IN` statements, each +within the limit, and their rows are joined before grouping, so the result is the same. On D1 +a level with 250 keys sends three statements. Keep the top level paged with `limit`. ## Tenant scope on every level diff --git a/src/database/relations.ts b/src/database/relations.ts index d5fae05..72dcb6e 100644 --- a/src/database/relations.ts +++ b/src/database/relations.ts @@ -7,7 +7,7 @@ // See design/relations.md for why this and not JSON aggregation. import { Effect } from "effect" -import type { CompiledQueryDecodeError } from "../ch/compile" +import { compileCH, type CompiledQueryDecodeError } from "../ch/compile" import type { QueryBuilderError } from "../ch/errors" import type { Condition, Expr } from "../ch/expr" import { from, type ColumnAccessor, type CHQuery } from "../ch/query" @@ -181,6 +181,12 @@ const distinct = (values: ReadonlyArray): Array => [ ...new Set(values.filter((value) => value !== null && value !== undefined)), ] +const chunks = (values: ReadonlyArray, size: number): Array> => { + const out: Array> = [] + for (let i = 0; i < values.length; i += size) out.push(values.slice(i, i + size)) + return out +} + const levelQuery = ( table: AnyTable, level: Level, @@ -232,16 +238,31 @@ const attach = ( ? distinct(rows.map((row) => row[parent.tenantColumn!])) : [] const tenantColumn = target.tenantColumn - const query = levelQuery( - target, - nested, - ($) => [ - $[relation.to].in_(...keys), - tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, - ], - crossTenant, - ) - const decoded = (yield* db.run(query as any)) as ReadonlyArray + const queryFor = (batch: ReadonlyArray) => + levelQuery( + target, + nested, + ($) => [ + $[relation.to].in_(...batch), + tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, + ], + crossTenant, + ) + // A dialect with a bound-value limit (D1: 100) would refuse a long + // `IN` list. Split the keys so each statement fits next to the + // level's other bindings; each parent key lands in one batch, so + // per-parent order is kept. + let batches: ReadonlyArray> = [keys] + const limit = db.dialect.params._tag === "bind" ? db.dialect.params.maxParameters : undefined + if (limit !== undefined && keys.length > 1) { + const one = yield* compileCH(queryFor(keys.slice(0, 1)) as any, undefined, { dialect: db.dialect }) + const size = Math.max(1, limit - (one.parameters.length - 1)) + if (keys.length > size) batches = chunks(keys, size) + } + const decoded: Array = [] + for (const batch of batches) { + decoded.push(...((yield* db.run(queryFor(batch) as any)) as ReadonlyArray)) + } children = yield* attach(tables, relations, relation.target, decoded, nested, crossTenant) } const byKey = new Map>() diff --git a/src/sqlite/relations.test.ts b/src/sqlite/relations.test.ts new file mode 100644 index 0000000..bd91e45 --- /dev/null +++ b/src/sqlite/relations.test.ts @@ -0,0 +1,110 @@ +// `findMany` on SQLite (node:sqlite) and on a local D1 binding. D1 binds at +// most 100 values per statement, so a level with more keys than that is split +// into several `IN (...)` statements. +import * as D1Client from "@effect/sql-d1/D1Client" +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Effect } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { Miniflare } from "miniflare" +import { expect, it } from "vitest" +import * as Db from "../database" +import * as SQLite from "../sqlite" + +const Units = SQLite.table("units", { id: SQLite.integer, org_id: SQLite.text, name: SQLite.text }, { tenantColumn: "org_id" }) +const Locks = SQLite.table("locks", { id: SQLite.integer, org_id: SQLite.text, unit_id: SQLite.integer }, { tenantColumn: "org_id" }) +const schema = Db.defineRelations({ units: Units, locks: Locks }, (r) => ({ + units: { locks: r.many("locks", { from: "id", to: "unit_id" }) }, + locks: { unit: r.one("units", { from: "unit_id", to: "id" }) }, +})) + +const setup = (db: Db.DatabaseApi, units: number) => + Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE units (id INTEGER PRIMARY KEY, org_id TEXT NOT NULL, name TEXT NOT NULL)" }) + yield* db.execute({ sql: "CREATE TABLE locks (id INTEGER PRIMARY KEY, org_id TEXT NOT NULL, unit_id INTEGER NOT NULL)" }) + const values = Array.from({ length: units }, (_, i) => `(${i + 1}, 'a', 'u${i + 1}')`).join(", ") + yield* db.execute({ sql: `INSERT INTO units VALUES ${values}, (${units + 1}, 'b', 'other')` }) + // Two locks per unit of org a, and one lock of org b pointing at unit 1. + const locks = Array.from({ length: units }, (_, i) => `(${2 * i + 1}, 'a', ${i + 1}), (${2 * i + 2}, 'a', ${i + 1})`).join(", ") + yield* db.execute({ sql: `INSERT INTO locks VALUES ${locks}, (${2 * units + 1}, 'b', 1)` }) + }) + +const read = Db.findMany(schema.units, { + where: ($) => [$.org_id.eq("a")], + orderBy: ($) => [[$.id, "asc"]], + with: { locks: { orderBy: ($) => [[$.id, "desc"]], with: { unit: true } } }, +}) + +it("loads nested relations on SQLite, pinned to the tenant, one statement per level", async () => { + const { units, statements } = await Effect.runPromise( + Effect.gen(function* () { + const statements: Array = [] + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { + dialect: SQLite.sqliteDialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + yield* setup(db, 2) + statements.length = 0 + const units = yield* read.pipe(Effect.provideService(Db.Database, db)) + return { units, statements } + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" }))), + ) + expect(units).toEqual([ + { + id: 1, + org_id: "a", + name: "u1", + locks: [ + { id: 2, org_id: "a", unit_id: 1, unit: { id: 1, org_id: "a", name: "u1" } }, + { id: 1, org_id: "a", unit_id: 1, unit: { id: 1, org_id: "a", name: "u1" } }, + ], + }, + { + id: 2, + org_id: "a", + name: "u2", + locks: [ + { id: 4, org_id: "a", unit_id: 2, unit: { id: 2, org_id: "a", name: "u2" } }, + { id: 3, org_id: "a", unit_id: 2, unit: { id: 2, org_id: "a", name: "u2" } }, + ], + }, + ]) + expect(statements).toHaveLength(3) + expect(statements[1]).toMatch(/"unit_id" IN \(/) + expect(statements[1]).toMatch(/"org_id" = \?/) +}) + +it("splits a level's IN list to fit D1's 100 bound values", async () => { + const mf = new Miniflare({ + modules: true, + script: "export default { fetch() { return new Response('D1 relations test') } }", + compatibilityDate: "2026-07-01", + d1Databases: ["DB"], + }) + try { + const binding = await mf.getD1Database("DB") + const { units, statements } = await Effect.runPromise( + Effect.gen(function* () { + const statements: Array = [] + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { + dialect: SQLite.d1Dialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + yield* setup(db, 250) + statements.length = 0 + const units = yield* read.pipe(Effect.provideService(Db.Database, db)) + return { units, statements } + }).pipe(Effect.provide(D1Client.layer({ db: binding }))), + ) + expect(units).toHaveLength(250) + expect(units.every((unit) => unit.locks.length === 2 && unit.locks.every((lock) => lock.unit?.id === unit.id))).toBe(true) + expect(units[0]!.locks.map((lock) => lock.id)).toEqual([2, 1]) + // 250 unit keys: 99 per statement next to the tenant pin is 3 statements; + // the 250 distinct unit_id keys of the locks level split the same way. + expect(statements).toHaveLength(7) + for (const statement of statements) expect((statement.match(/\?/g) ?? []).length).toBeLessThanOrEqual(100) + } finally { + await mf.dispose() + } +}, 60_000) From 95e52405e1f9aa88ab0a583eec83ec79da0a841c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 2 Nov 2026 16:36:40 +0100 Subject: [PATCH 9/9] test(database): relation filters and _count on SQLite and D1 Correlated EXISTS, count() and _count run on node:sqlite inside the outer tenant, and a 250-key _count level on D1 splits into statements of at most 100 bindings. Docs and changeset name SQLite and D1. Co-Authored-By: Claude Opus 5.5 --- .changeset/relation-filters.md | 2 +- docs/relations.md | 11 +++- src/sqlite/relations.test.ts | 91 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 2 deletions(-) diff --git a/.changeset/relation-filters.md b/.changeset/relation-filters.md index cec33db..f321f81 100644 --- a/.changeset/relation-filters.md +++ b/.changeset/relation-filters.md @@ -2,4 +2,4 @@ "@maple-dev/effect-orm": minor --- -Relation filters and counts in `/database`, as Prisma's `some` / `none` / `every` and `_count`. `findMany`'s `where` gets the relations of its level as a second argument: `where: ($, r) => [r.posts.some(($) => [$.published.eq(false)])]`. `Db.related(schema.users, $)` gives the same filters on a builder query, and `r.posts.count(where?)`, an `Expr` for a `select` or a `where`. The filters are correlated `EXISTS` / `NOT EXISTS` subqueries. `_count: { posts: true }` in `findMany` adds `_count: { posts: number }` to each row, with one grouped statement per level. When both tables declare a tenant column, the subquery matches the outer row's tenant, and a filter under a scoped query stays `"single-tenant"`. See docs/relations.md. +Relation filters and counts in `/database`, as Prisma's `some` / `none` / `every` and `_count`. `findMany`'s `where` gets the relations of its level as a second argument: `where: ($, r) => [r.posts.some(($) => [$.published.eq(false)])]`. `Db.related(schema.users, $)` gives the same filters on a builder query, and `r.posts.count(where?)`, an `Expr` for a `select` or a `where`. The filters are correlated `EXISTS` / `NOT EXISTS` subqueries. `_count: { posts: true }` in `findMany` adds `_count: { posts: number }` to each row, with one grouped statement per level. When both tables declare a tenant column, the subquery matches the outer row's tenant, and a filter under a scoped query stays `"single-tenant"`. Works on Postgres, ClickHouse 25+, SQLite and D1; a `_count` level is split to the dialect's bound-value limit like a level of `with`. See docs/relations.md. diff --git a/docs/relations.md b/docs/relations.md index fb8a109..aa6e6a0 100644 --- a/docs/relations.md +++ b/docs/relations.md @@ -179,7 +179,8 @@ Two ways, for two shapes of read: only the rows that match. Only a `many` relation can be counted. It runs one statement per counted relation, the same way as a level of `with`: `SELECT , count(*) FROM WHERE IN () [AND ] GROUP BY `. - A row with no related rows gets `0`. + A row with no related rows gets `0`. Its keys are split to the bound-value limit like a level + of `with`. - **`r.posts.count(where?)` on a query**, an `Expr`: a correlated `coalesce((SELECT count(*) …), 0)`. Use it in a builder `select`, or in a `where` such as `Db.related(schema.users, $).posts.count().gte(2)`. @@ -220,6 +221,14 @@ on by default in 26.8). On 26.8: _(Backed by `tests/relation-filters.clickhouse.test.ts`.)_ +## On SQLite and D1 + +The filters, `count()` and `_count` are plain correlated `EXISTS` and `count(*)`, so they run on +SQLite and D1 as on Postgres, with the same tenant rule. Each filter binds its own `where` values +inside the outer statement, so they count toward D1's 100 bound values. + +_(Backed by `src/sqlite/relations.test.ts`.)_ + ## Not yet - Choosing columns per level (`columns: { id: true }`). Every level reads every column. diff --git a/src/sqlite/relations.test.ts b/src/sqlite/relations.test.ts index bd91e45..e7b4bd6 100644 --- a/src/sqlite/relations.test.ts +++ b/src/sqlite/relations.test.ts @@ -108,3 +108,94 @@ it("splits a level's IN list to fit D1's 100 bound values", async () => { await mf.dispose() } }, 60_000) + +// Relation filters and counts: correlated EXISTS and count(*) are plain SQL, so +// SQLite runs them as Postgres does. `_count` is a grouped level, split on D1 +// like a loaded one. +const withSqlite = (units: number, body: (db: Db.DatabaseApi) => Effect.Effect) => + Effect.runPromise( + Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect, requireTenantScope: true }) + yield* setup(db, units) + yield* db.execute({ sql: "DELETE FROM locks WHERE unit_id = 2" }) + return yield* body(db).pipe(Effect.provideService(Db.Database, db)) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" }))), + ) + +it("filters by related rows and counts them on SQLite, inside the outer tenant", async () => { + const seen = await withSqlite(3, (db) => + Effect.gen(function* () { + const some = yield* Db.findMany(schema.units, { + where: ($, r) => [$.org_id.eq("a"), r.locks.some()], + orderBy: ($) => [[$.id, "asc"]], + }) + const none = yield* Db.findMany(schema.units, { where: ($, r) => [$.org_id.eq("a"), r.locks.none()] }) + const every = yield* Db.findMany(schema.units, { + where: ($, r) => [$.org_id.eq("a"), r.locks.every(($) => [$.id.lt(4)])], + orderBy: ($) => [[$.id, "asc"]], + }) + const counted = yield* Db.findMany(schema.units, { + where: ($) => [$.org_id.eq("a")], + orderBy: ($) => [[$.id, "asc"]], + _count: { locks: true }, + }) + const selected = yield* db.run( + SQLite.from(Units) + .select(($) => ({ id: $.id, org_id: $.org_id, locks: Db.related(schema.units, $).locks.count() })) + .where(($) => [$.org_id.eq("a")]) + .orderBy(["id", "asc"]), + ) + return { + some: some.map((unit) => unit.id), + none: none.map((unit) => unit.id), + every: every.map((unit) => unit.id), + counted: counted.map((unit) => [unit.id, unit._count.locks]), + selected: selected.map((row) => [row.id, row.locks]), + } + }), + ) + // Unit 1 also has a lock of org b: the tenant pin keeps it out of every filter and count. + expect(seen).toEqual({ + some: [1, 3], + none: [2], + every: [1, 2], + counted: [[1, 2], [2, 0], [3, 2]], + selected: [[1, 2], [2, 0], [3, 2]], + }) +}) + +it("splits a _count level to fit D1's 100 bound values", async () => { + const mf = new Miniflare({ + modules: true, + script: "export default { fetch() { return new Response('D1 relation counts test') } }", + compatibilityDate: "2026-07-01", + d1Databases: ["DB"], + }) + try { + const binding = await mf.getD1Database("DB") + const { units, statements } = await Effect.runPromise( + Effect.gen(function* () { + const statements: Array = [] + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { + dialect: SQLite.d1Dialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + yield* setup(db, 250) + statements.length = 0 + const units = yield* Db.findMany(schema.units, { + where: ($, r) => [$.org_id.eq("a"), r.locks.some(($) => [$.id.gt(0)])], + _count: { locks: true }, + }).pipe(Effect.provideService(Db.Database, db)) + return { units, statements } + }).pipe(Effect.provide(D1Client.layer({ db: binding }))), + ) + expect(units).toHaveLength(250) + expect(units.every((unit) => unit._count.locks === 2)).toBe(true) + // The top level, then 3 grouped count statements of at most 99 keys. + expect(statements).toHaveLength(4) + for (const statement of statements) expect((statement.match(/\?/g) ?? []).length).toBeLessThanOrEqual(100) + } finally { + await mf.dispose() + } +}, 60_000)