diff --git a/.changeset/table-row-schemas.md b/.changeset/table-row-schemas.md new file mode 100644 index 0000000..774ebe3 --- /dev/null +++ b/.changeset/table-row-schemas.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`tableSchemas(table)` derives the Effect Schemas of a table's rows from its columns, on `/clickhouse`, `/postgres` and `/sqlite`: `select` (every column, decoded from the wire), `insert` (computed columns left out, defaulted and nullable ones optional), `update` (every writable column optional), and `json`, `jsonCreate` and `jsonUpdate`, the same rows as plain JSON for an HTTP API. The names follow `Model.Class`. A decoded `insert` or `update` row is a row `insertInto(...).values` or `update(...).set` takes, so request bodies and repositories no longer need Schemas written by hand next to the table. diff --git a/docs/reference.md b/docs/reference.md index fe8f4eb..c867e29 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -47,6 +47,7 @@ Everything in this section is exported, identically, from `/clickhouse`, `/postg | `unionAll` | `(...queries) => CHUnionQuery` | | `update` | `(table) => CHUpdateStart`, then `CHUpdate`: `.set(record \| fn)`, `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)`. See [Updating and deleting](./updates-and-deletes.md) | | `deleteFrom` | `(table) => CHDelete`: `.where(fn)` or `.allRows()`, `.returning(...)`, `.settings(record)` | +| `tableSchemas` | `(table) => TableSchemas`: the `select`, `insert`, `update`, `json`, `jsonCreate` and `jsonUpdate` row Schemas. See [Row Schemas](./tables-and-types.md#row-schemas) | | `insertInto` | `(table) => CHInsertStart`, then `CHInsert`; `.values(row \| rows)` or `.select(query)` sets its rows, `.settings(record)` ClickHouse `SETTINGS`, `.returning(...)` the RETURNING list, `.onConflictDoNothing(options?)` / `.onConflictDoUpdate(options)` the ON CONFLICT clause (Postgres and SQLite). See [Inserting rows](./inserts.md) | Tables come from the dialect's `table`: [`CH.table`](#tables-and-ddl) or @@ -180,7 +181,8 @@ decoded type of a column), `InferEncoded` (its wire type), `ColumnDefs`, `Nullab `OutputToColumnDefs`. **Inference** — `InferOutput`, `InferQueryOutput`, `InferUnionOutput`, `SelectRowOf`, -`InsertRow`, `InsertRowOf`, `UpdateSet`, `UpdateSetOf`. +`InsertRow`, `InsertRowOf`, `UpdateSet`, `UpdateSetOf`, `TableSchemas` (what `tableSchemas` +returns for a table). **What `compile` checks** — `NeedsSelect` (the query has a SELECT list) and `ParamsSatisfied` (the params object fills every required param). A helper generic diff --git a/docs/tables-and-types.md b/docs/tables-and-types.md index 23b0795..c191f8d 100644 --- a/docs/tables-and-types.md +++ b/docs/tables-and-types.md @@ -251,6 +251,36 @@ Related utilities: `ColumnDefs` (the shape of a `columns` record), `OutputToColu (converts a query's output row back into column defs, used by `fromQuery`), and `NullableColumnDefs` (what `leftJoin` applies to the joined side). +## Row Schemas + +`tableSchemas(table)` derives the Effect Schemas of a table's rows from its columns, so an +`HttpApi` endpoint or a repository uses the same definition as the queries. The variant names are +the ones `Model.Class` uses: + +| Variant | Columns | Decodes from | +| ------------ | ---------------------------------------------------- | ------------------ | +| `select` | every column | the database wire | +| `insert` | no computed column; defaulted and nullable optional | the database wire | +| `update` | no computed column; every column optional | the database wire | +| `json` | as `select` | JSON | +| `jsonCreate` | as `insert` | JSON | +| `jsonUpdate` | as `update` | JSON | + +The JSON variants are `Schema.toCodecJson` of each row's type, so a `DateTime` is an ISO string +and a branded column keeps its brand. An optional key also takes `undefined`, as in an insert row. +`tableSchemas` is on every entry; on a SQLite table `select` and `insert` read SQLite's wire forms +(`0`/`1` booleans, timestamp and JSON text), and the JSON variants read plain JSON. + +```ts +const { select, jsonCreate } = CH.tableSchemas(Requests) + +type Request = typeof select.Type // SelectRowOf +const body = Schema.decodeUnknownSync(jsonCreate)({ OrgId: "org_1", Route: "/" }) +CH.insertInto(Requests).values(body) // a decoded body is an insert row +``` + +_(Backed by `docs/tables-and-types.md > Row Schemas`.)_ + ## Map columns `Map` columns get a `.get(key)` accessor that compiles to ClickHouse's bracket syntax: diff --git a/src/ch/table-schemas.test-d.ts b/src/ch/table-schemas.test-d.ts new file mode 100644 index 0000000..f6f9b10 --- /dev/null +++ b/src/ch/table-schemas.test-d.ts @@ -0,0 +1,80 @@ +// Type-level tests: the row Schemas of a table + +import { type DateTime, Schema } from "effect" +import { expectTypeOf } from "expect-type" +import * as CH from "./index" +import * as CHD from "../clickhouse" +import * as SQLite from "../sqlite" + +const Spans = CHD.table("spans", { + columns: { + OrgId: CH.string, + Id: CHD.column(CH.uint64, { default: 0 }), + At: CH.dateTime, + Note: CH.nullable(CH.string), + Day: CHD.column(CH.string, { materialized: "toString(toDate(At))" }), + }, + engine: CHD.engine.mergeTree(), + orderBy: ["OrgId", "At"], +}) + +const schemas = CH.tableSchemas(Spans) + +// select is the whole decoded row, as SelectRowOf names it. +expectTypeOf().toEqualTypeOf>() + +// insert: defaulted and nullable columns optional, computed ones absent. +expectTypeOf().toEqualTypeOf<{ + readonly OrgId: string + readonly At: DateTime.Utc + readonly Id?: number | undefined + readonly Note?: string | null | undefined +}>() + +// A decoded insert row is a row `values` takes. +declare const created: typeof schemas.insert.Type +CH.insertInto(Spans).values(created) + +// update: every writable column optional. +expectTypeOf().toEqualTypeOf<{ + readonly OrgId?: string | undefined + readonly Id?: number | undefined + readonly At?: DateTime.Utc | undefined + readonly Note?: string | null | undefined +}>() +declare const patch: typeof schemas.update.Type & { readonly Id: number } +CH.update(Spans).set(patch).allRows() + +// The json variants keep the row types and encode to JSON. +expectTypeOf().toEqualTypeOf>() +expectTypeOf().toEqualTypeOf() +expectTypeOf().toEqualTypeOf() +expectTypeOf().toEqualTypeOf() + +// The select Schema's wire side is what the database sends. +expectTypeOf().toEqualTypeOf() + +// A table written with the bare `table` and no options: every non-null column required. +const Plain = CH.table("plain", { A: CH.string, B: CH.nullable(CH.uint32) }) +expectTypeOf["insert"]["Type"]>().toEqualTypeOf<{ + readonly A: string + readonly B?: number | null | undefined +}>() + +// A computed column is not part of an insert row. +// @ts-expect-error Day is computed +const withDay: typeof schemas.insert.Type = { OrgId: "o", At: {} as DateTime.Utc, Day: "2026-01-01" } + +// A SQLite table: a defaulted column is optional, and a decoded insert row is a row `values` takes. +const Notes = SQLite.table("notes", { + columns: { id: SQLite.integer, pinned: SQLite.column(SQLite.boolean, { default: false }), at: SQLite.timestamp }, + primaryKey: ["id"], +}) +const notes = SQLite.tableSchemas(Notes) +expectTypeOf().toEqualTypeOf<{ + readonly id: number + readonly at: DateTime.Utc + readonly pinned?: boolean | undefined +}>() +declare const note: typeof notes.insert.Type +SQLite.insertInto(Notes).values(note) diff --git a/src/ch/table-schemas.test.ts b/src/ch/table-schemas.test.ts new file mode 100644 index 0000000..ead5562 --- /dev/null +++ b/src/ch/table-schemas.test.ts @@ -0,0 +1,163 @@ +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { describe, expect, it } from "@effect/vitest" +import { DateTime, Effect, Schema } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as CH from "./index" +import * as CHD from "../clickhouse" +import * as Db from "../database" +import * as PG from "../postgres" +import { renderSqliteSchema, sqliteEntitiesOf } from "../schema" +import * as SQLite from "../sqlite" + +const Spans = CHD.table("spans", { + columns: { + OrgId: CH.string, + Id: CHD.column(CH.uint64, { default: 0 }), + At: CH.dateTime, + Note: CH.nullable(CH.string), + Day: CHD.column(CH.string, { materialized: "toString(toDate(At))" }), + }, + engine: CHD.engine.mergeTree(), + orderBy: ["OrgId", "At"], +}) + +describe("tableSchemas", () => { + it("select decodes a whole row from the wire, as a SELECT of every column does", () => { + const { select } = CH.tableSchemas(Spans) + const row = Schema.decodeUnknownSync(select)({ + OrgId: "o", + Id: "7", + At: "2026-01-01 00:00:00", + Note: null, + Day: "2026-01-01", + }) + expect(row).toEqual({ + OrgId: "o", + Id: 7, + At: DateTime.makeUnsafe("2026-01-01T00:00:00Z"), + Note: null, + Day: "2026-01-01", + }) + }) + + it("insert makes defaulted and nullable columns optional and leaves computed ones out", () => { + const { insert } = CH.tableSchemas(Spans) + expect(Object.keys(insert.fields)).toEqual(["OrgId", "Id", "At", "Note"]) + const decode = Schema.decodeUnknownSync(insert) + expect(decode({ OrgId: "o", At: "2026-01-01 00:00:00" })).toEqual({ + OrgId: "o", + At: DateTime.makeUnsafe("2026-01-01T00:00:00Z"), + }) + expect(decode({ OrgId: "o", At: "2026-01-01 00:00:00", Id: undefined, Note: null })).toEqual({ + OrgId: "o", + At: DateTime.makeUnsafe("2026-01-01T00:00:00Z"), + Id: undefined, + Note: null, + }) + expect(() => decode({ OrgId: "o" })).toThrow(/At/) + }) + + it("update makes every writable column optional and leaves computed ones out", () => { + const { update } = CH.tableSchemas(Spans) + expect(Object.keys(update.fields)).toEqual(["OrgId", "Id", "At", "Note"]) + const decode = Schema.decodeUnknownSync(update) + expect(decode({})).toEqual({}) + expect(decode({ Id: "8", Note: null })).toEqual({ Id: 8, Note: null }) + expect(() => decode({ OrgId: null })).toThrow(/OrgId/) + }) + + it("json writes the select row in plain JSON and reads it back", () => { + const { json } = CH.tableSchemas(Spans) + const row = { OrgId: "o", Id: 7, At: DateTime.makeUnsafe("2026-01-01T00:00:00Z"), Note: null, Day: "2026-01-01" } + const encoded = Schema.encodeSync(json)(row) + expect(encoded).toEqual({ OrgId: "o", Id: 7, At: "2026-01-01T00:00:00.000Z", Note: null, Day: "2026-01-01" }) + expect(Schema.decodeUnknownSync(json)(JSON.parse(JSON.stringify(encoded)))).toEqual(row) + }) + + it("jsonCreate and jsonUpdate read request bodies with the insert and update keys", () => { + const { jsonCreate, jsonUpdate } = CH.tableSchemas(Spans) + expect(Schema.decodeUnknownSync(jsonCreate)({ OrgId: "o", At: "2026-01-01T00:00:00.000Z", Note: null })).toEqual({ + OrgId: "o", + At: DateTime.makeUnsafe("2026-01-01T00:00:00Z"), + Note: null, + }) + expect(() => Schema.decodeUnknownSync(jsonCreate)({ OrgId: "o" })).toThrow(/At/) + expect(Schema.decodeUnknownSync(jsonUpdate)({ Note: "n" })).toEqual({ Note: "n" }) + expect(Schema.decodeUnknownSync(jsonUpdate)({ Day: "2026-01-01" })).toEqual({}) + }) + + it("reads a Postgres table: identity and default columns are optional on insert, keys are the column keys", () => { + const Requests = PG.table("requests", { + columns: { + id: PG.column(PG.int8, { identity: "always" }), + orgId: PG.column(PG.text, { name: "org_id" }), + createdAt: PG.column(PG.timestamptz, { defaultExpr: "now()" }), + body: PG.nullable(PG.jsonb<{ readonly path: string }>()), + }, + primaryKey: ["id"], + }) + const { insert, json } = PG.tableSchemas(Requests) + expect(Schema.decodeUnknownSync(insert)({ orgId: "o" })).toEqual({ orgId: "o" }) + expect(() => Schema.decodeUnknownSync(insert)({})).toThrow(/orgId/) + expect( + Schema.encodeSync(json)({ + id: 1, + orgId: "o", + createdAt: DateTime.makeUnsafe("2026-01-01T00:00:00Z"), + body: { path: "/" }, + }), + ).toEqual({ id: 1, orgId: "o", createdAt: "2026-01-01T00:00:00.000Z", body: { path: "/" } }) + }) + + // SQLite stores booleans as 0/1 and timestamps and JSON as text, so `select` and `insert` read + // those wire forms while `json` / `jsonCreate` read plain JSON. + const Notes = SQLite.table("notes", { + columns: { + id: SQLite.integer, + orgId: SQLite.column(SQLite.text, { name: "org_id" }), + pinned: SQLite.column(SQLite.boolean, { default: false }), + at: SQLite.timestamp, + meta: SQLite.nullable(SQLite.json(Schema.Struct({ tag: Schema.String }))), + }, + primaryKey: ["id"], + }) + const at = DateTime.makeUnsafe("2026-01-01T00:00:00.250Z") + + it("reads a SQLite table: wire forms on select and insert, plain JSON on the json variants", () => { + const { select, insert, json, jsonCreate } = SQLite.tableSchemas(Notes) + expect(Schema.decodeUnknownSync(select)({ id: 1, orgId: "o", pinned: 1, at: "2026-01-01 00:00:00.250", meta: '{"tag":"a"}' })).toEqual({ + id: 1, + orgId: "o", + pinned: true, + at, + meta: { tag: "a" }, + }) + expect(Schema.decodeUnknownSync(insert)({ id: 1, orgId: "o", at: "2026-01-01 00:00:00.250" })).toEqual({ id: 1, orgId: "o", at }) + expect(() => Schema.decodeUnknownSync(insert)({ id: 1.5, orgId: "o", at: "2026-01-01 00:00:00" })).toThrow(/id/) + expect(Schema.encodeSync(json)({ id: 1, orgId: "o", pinned: true, at, meta: { tag: "a" } })).toEqual({ + id: 1, + orgId: "o", + pinned: true, + at: "2026-01-01T00:00:00.250Z", + meta: { tag: "a" }, + }) + expect(Schema.decodeUnknownSync(jsonCreate)({ id: 1, orgId: "o", at: "2026-01-01T00:00:00.250Z", meta: null })).toEqual({ + id: 1, + orgId: "o", + at, + meta: null, + }) + }) + + it.effect("a SQLite jsonCreate body inserts, and select decodes the row read back", () => + Effect.gen(function* () { + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { dialect: SQLite.sqliteDialect }) + for (const sql of yield* Effect.map(sqliteEntitiesOf([Notes]), renderSqliteSchema)) yield* db.execute({ sql }) + const { jsonCreate } = SQLite.tableSchemas(Notes) + const body = Schema.decodeUnknownSync(jsonCreate)({ id: 1, orgId: "o", at: "2026-01-01T00:00:00.250Z", meta: { tag: "a" } }) + yield* db.run(SQLite.insertInto(Notes).values(body)) + const rows = yield* db.run(SQLite.from(Notes).select()) + expect(rows).toEqual([{ id: 1, orgId: "o", pinned: false, at, meta: { tag: "a" } }]) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" }))), + ) +}) diff --git a/src/ch/table-schemas.ts b/src/ch/table-schemas.ts new file mode 100644 index 0000000..e5ccfea --- /dev/null +++ b/src/ch/table-schemas.ts @@ -0,0 +1,76 @@ +// Table Schemas +// +// The Effect Schemas of a table's rows, derived from its columns: what a +// `SELECT` of every column decodes, what an insert or an update takes, and the +// JSON forms of the same for an HTTP API. The names follow `Model.Class` in +// `effect/unstable/schema/Model`, so a `SqlModel` user finds the same variants. + +import { Schema } from "effect" +import type { Table } from "./table" +import type { ColumnDefs, InferTS } from "./types" + +/** A bare `string` means the table did not say, which is read as none. */ +type Known = string extends K ? never : K + +type Simplify = { [K in keyof A]: A[K] } & {} + +type NullableKeys = { + [K in keyof Cols]: null extends InferTS ? K : never +}[keyof Cols] & + string + +type Writable = Exclude> +type Optional = Extract< + Known | NullableKeys, + Writable +> + +type SelectFields = { readonly [K in keyof Cols]: Cols[K]["schema"] } + +type InsertFields = Simplify< + { + readonly [K in Exclude, Optional>]: Cols[K]["schema"] + } & { + readonly [K in Optional]: Schema.optional + } +> + +type UpdateFields = { + readonly [K in Writable]: Schema.optional +} + +/** The row Schemas of a table value: `TableSchemas`. */ +export type TableSchemas = + T extends Table + ? { + readonly select: Schema.Struct> + readonly insert: Schema.Struct> + readonly update: Schema.Struct> + readonly json: Schema.toCodecJson>>> + readonly jsonCreate: Schema.toCodecJson>>> + readonly jsonUpdate: Schema.toCodecJson>>> + } + : never + +type AnyTable = Table + +export const tableSchemas = (table: T): TableSchemas => { + const computed = new Set(table.computed ?? []) + const defaulted = new Set(table.defaults ?? []) + const columns = Object.entries(table.columns) + const writable = columns.filter(([key]) => !computed.has(key)) + const isOptional = (key: string, schema: Schema.Top) => defaulted.has(key) || Schema.is(schema)(null) + const select = Object.fromEntries(columns.map(([key, column]) => [key, column.schema])) + const insert = Object.fromEntries( + writable.map(([key, column]) => [key, isOptional(key, column.schema) ? Schema.optional(column.schema) : column.schema]), + ) + const update = Object.fromEntries(writable.map(([key, column]) => [key, Schema.optional(column.schema)])) + const rows = { select: Schema.Struct(select), insert: Schema.Struct(insert), update: Schema.Struct(update) } + const toJson = (schema: Schema.Struct) => Schema.toCodecJson(Schema.toType(schema)) + return { + ...rows, + json: toJson(rows.select), + jsonCreate: toJson(rows.insert), + jsonUpdate: toJson(rows.update), + } as unknown as TableSchemas +} diff --git a/src/core.ts b/src/core.ts index b32adcf..a68e477 100644 --- a/src/core.ts +++ b/src/core.ts @@ -17,6 +17,7 @@ export { // Tables, as the query builder sees them. Define one with the dialect's `table`. export { type SelectRowOf, type Table } from "./ch/table" +export { type TableSchemas, tableSchemas } from "./ch/table-schemas" // Core expression primitives export { diff --git a/src/docs-examples.test.ts b/src/docs-examples.test.ts index ef673be..d4f01f2 100644 --- a/src/docs-examples.test.ts +++ b/src/docs-examples.test.ts @@ -192,6 +192,28 @@ describe("docs/tables-and-types.md", () => { "SELECT counters.Hits AS hits FROM counters WHERE counters.OrgId = 'org_123' AND counters.Live = 1", ) }) + + it("Row Schemas", () => { + const Requests = CHE.table("requests", { + columns: { + OrgId: CHE.string, + Timestamp: CHE.column(CHE.dateTime, { defaultExpr: "now()" }), + Route: CHE.string, + Status: CHE.column(CHE.uint16, { default: 200 }), + Hour: CHE.column(CHE.dateTime, { materialized: "toStartOfHour(Timestamp)" }), + }, + engine: CHE.engine.mergeTree(), + orderBy: ["OrgId", "Route", "Timestamp"], + }) + const { select, jsonCreate } = CH.tableSchemas(Requests) + + const body = Schema.decodeUnknownSync(jsonCreate)({ OrgId: "org_1", Route: "/" }) + expect(body).toEqual({ OrgId: "org_1", Route: "/" }) + expect(oneLine(compileCHUnsafe(CH.insertInto(Requests).values(body)).sql)).toBe( + "INSERT INTO requests (OrgId, Route) VALUES ('org_1', '/')", + ) + expect(Object.keys(select.fields)).toEqual(["OrgId", "Timestamp", "Route", "Status", "Hour"]) + }) }) // queries.md