From e339a38823d07b54319218b4f1fea8ff13f1e45b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 28 Jul 2026 23:52:43 -0700 Subject: [PATCH 1/2] fix(db): reconcile folders stranded by old pods during the cutover drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0272 and 0274 both ran while the previous app version was still serving. During the blue/green window the old pods write the legacy folder tables exclusively while the new pods write `folder` exclusively, so any folder an old pod creates between the migration and its own termination exists only in the legacy table and is invisible to the new code. 0274 asked to be re-run once the deploy drained; a journaled migration never replays, so the re-run needs its own file. Covers both trees, where 0274 covered only `file`. The workflow side is the higher-volume one and is where the stranded row actually landed in production. Insert-only for ids missing from `folder`, never a mirror upsert: 0272's dedup renamed 47 workflow folders, and an upsert would revert all 47 to their colliding names and violate the active-unique index doing it. Names are deduplicated against the first free suffix rather than a row number, since the target name may already be held by a row this block is not inserting. Only active rows dedupe — the partial index ignores soft-deleted rows, so an archived folder keeps its name and reappears under it if restored. Rows insert in depth order so a parent always precedes its children, which the self-referencing FK and the parent-resource-type trigger both require. Wrapped in a DO block for the same reason 0274 is: 0272 ends in an embedded COMMIT, so files after it are not guaranteed to run inside drizzle's batch transaction. Verified by executing the file against Postgres 17 with adversarial fixtures: a 0272-renamed row is left untouched, a stranded active row whose name and whose "(2)" are both taken lands on "(3)", a stranded archived row keeps its colliding name, a fully-stranded three-level chain inserts parents first, sibling dedup scopes to the parent, and cross-workspace and cross-resource-type names do not interfere. Idempotent across three consecutive runs with byte-identical state. Production at 06:52Z has 1 row to insert (archived, empty, no parent) on the workflow side and 0 on the file side. --- ...75_folder_cutover_post_drain_reconcile.sql | 136 ++++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + 2 files changed, 143 insertions(+) create mode 100644 packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql diff --git a/packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql b/packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql new file mode 100644 index 00000000000..fedc4f9e36f --- /dev/null +++ b/packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql @@ -0,0 +1,136 @@ +-- Post-drain reconcile for the generic-folder cutover, covering BOTH legacy trees. +-- +-- 0272 mirrored `workflow_folder` and `workspace_file_folders` into `folder`, and 0274 +-- reconciled the file tree. Both ran while the previous app version was still serving. During +-- the blue/green rolling window the old pods keep writing the LEGACY tables exclusively while +-- the new pods write `folder` exclusively, so every folder an old pod created between the +-- migration and its own termination exists only in the legacy table and is invisible to the +-- new code. 0274 anticipated this and asked to be run again once the deploy drained; a +-- journaled migration never replays, so that re-run has to be its own file. This is it. +-- +-- Production measurement at 2026-07-29T06:40Z, mid-drain: exactly 1 stranded row +-- (`workflow_folder`, archived, no parent, no workflows inside) and 0 on the file side, with 0 +-- name collisions and 0 absent parents on both trees. The count at execution time may be +-- higher, since old pods were still serving when that was taken. +-- +-- INSERT-ONLY, NOT A MIRROR. 0274 could upsert every column because file folders can never +-- diverge in name — `workspace_file_folders` enforces the same active-unique key `folder` +-- does. That does NOT hold here: 0272's dedup renamed 47 workflow folders across 35 collision +-- groups, so `folder` is deliberately different from `workflow_folder` for those rows. An +-- upsert would revert all 47 to their colliding names and violate +-- `folder_workspace_resource_parent_name_active_unique` in the process. Only ids absent from +-- `folder` are touched; every row already mirrored is left exactly as it is. +-- +-- Names are deduplicated against the FIRST FREE suffix rather than a row number, because the +-- target name may already be taken by a row this block is not inserting. Only ACTIVE rows are +-- deduplicated — the partial unique index ignores soft-deleted rows, so an archived folder +-- keeps its original name and reappears under it if restored. +-- +-- Rows are inserted in depth order so a parent always lands before its children: the +-- self-referencing FK `folder_parent_id_folder_id_fk` is checked immediately, and the +-- `folder_parent_resource_type_match` trigger reads the parent row. A child inserted before +-- its parent would fail both. +-- +-- RUN THIS BEFORE THE SOFT-DELETE CLEANUP JOB PURGES EXPIRED `folder` ROWS. Cleanup +-- hard-deletes from `folder` but never from the legacy tables, so a run that lands after a +-- purge reinstates every purged folder as a soft-deleted phantom in Recently Deleted whose +-- files and workflows are already gone. The retention window is far longer than any drain, so +-- running promptly after the deploy is sufficient — but this is the one ordering constraint +-- that makes the file dangerous rather than merely redundant if deferred indefinitely. +-- +-- Idempotent and a clean no-op once reconciled: the driving queries select only ids missing +-- from `folder`, so a second execution finds nothing and the `ON CONFLICT (id) DO NOTHING` +-- guards against a concurrent writer inserting the same id between the SELECT and the INSERT. + +-- Wrapped in a DO block for the same reason 0274 is: 0272 ends with an embedded COMMIT for its +-- trailing CONCURRENTLY index builds, so files after it are not guaranteed to run inside +-- drizzle's batch transaction. Both trees must land together or not at all. +DO $$ +DECLARE + rec RECORD; + candidate TEXT; + suffix INT; +BEGIN + -- Workflow tree. `archived_at` is this table's soft-delete column; `folder` calls it + -- `deleted_at`. Depth is computed over the SOURCE table so a stranded child of a stranded + -- parent still orders correctly. + FOR rec IN + WITH RECURSIVE tree AS ( + SELECT s.id, 0 AS depth + FROM "workflow_folder" s + WHERE s."parent_id" IS NULL + UNION ALL + SELECT s.id, t.depth + 1 + FROM "workflow_folder" s + JOIN tree t ON t.id = s."parent_id" + ) + SELECT s."id", s."name", s."user_id", s."workspace_id", s."parent_id", + s."sort_order", s."created_at", s."updated_at", s."archived_at" AS deleted_at + FROM "workflow_folder" s + JOIN tree t ON t.id = s."id" + WHERE NOT EXISTS (SELECT 1 FROM "folder" f WHERE f."id" = s."id") + ORDER BY t.depth, s."created_at" + LOOP + candidate := rec."name"; + IF rec.deleted_at IS NULL THEN + suffix := 1; + WHILE EXISTS ( + SELECT 1 FROM "folder" f + WHERE f."workspace_id" = rec."workspace_id" + AND f."resource_type" = 'workflow' + AND coalesce(f."parent_id", '') = coalesce(rec."parent_id", '') + AND f."name" = candidate + AND f."deleted_at" IS NULL + ) LOOP + suffix := suffix + 1; + candidate := rec."name" || ' (' || suffix || ')'; + END LOOP; + END IF; + + INSERT INTO "folder" (id, resource_type, name, user_id, workspace_id, parent_id, locked, sort_order, created_at, updated_at, deleted_at) + VALUES (rec."id", 'workflow', candidate, rec."user_id", rec."workspace_id", rec."parent_id", + false, rec."sort_order", rec."created_at", rec."updated_at", rec.deleted_at) + ON CONFLICT (id) DO NOTHING; + END LOOP; + + -- File tree. Same shape; this table already uses `deleted_at`. 0274 reconciled it fully, so + -- this pass only picks up what old pods wrote after 0274 ran. + FOR rec IN + WITH RECURSIVE tree AS ( + SELECT s.id, 0 AS depth + FROM "workspace_file_folders" s + WHERE s."parent_id" IS NULL + UNION ALL + SELECT s.id, t.depth + 1 + FROM "workspace_file_folders" s + JOIN tree t ON t.id = s."parent_id" + ) + SELECT s."id", s."name", s."user_id", s."workspace_id", s."parent_id", + s."sort_order", s."created_at", s."updated_at", s."deleted_at" + FROM "workspace_file_folders" s + JOIN tree t ON t.id = s."id" + WHERE NOT EXISTS (SELECT 1 FROM "folder" f WHERE f."id" = s."id") + ORDER BY t.depth, s."created_at" + LOOP + candidate := rec."name"; + IF rec."deleted_at" IS NULL THEN + suffix := 1; + WHILE EXISTS ( + SELECT 1 FROM "folder" f + WHERE f."workspace_id" = rec."workspace_id" + AND f."resource_type" = 'file' + AND coalesce(f."parent_id", '') = coalesce(rec."parent_id", '') + AND f."name" = candidate + AND f."deleted_at" IS NULL + ) LOOP + suffix := suffix + 1; + candidate := rec."name" || ' (' || suffix || ')'; + END LOOP; + END IF; + + INSERT INTO "folder" (id, resource_type, name, user_id, workspace_id, parent_id, locked, sort_order, created_at, updated_at, deleted_at) + VALUES (rec."id", 'file', candidate, rec."user_id", rec."workspace_id", rec."parent_id", + false, rec."sort_order", rec."created_at", rec."updated_at", rec."deleted_at") + ON CONFLICT (id) DO NOTHING; + END LOOP; +END $$; diff --git a/packages/db/migrations/meta/_journal.json b/packages/db/migrations/meta/_journal.json index eea606ba98c..14a9dc4de53 100644 --- a/packages/db/migrations/meta/_journal.json +++ b/packages/db/migrations/meta/_journal.json @@ -1919,6 +1919,13 @@ "when": 1785281897202, "tag": "0274_file_folder_cutover_reconcile", "breakpoints": true + }, + { + "idx": 275, + "version": "7", + "when": 1785281897203, + "tag": "0275_folder_cutover_post_drain_reconcile", + "breakpoints": true } ] } From a5dac82cbdb26e76b32b24a3ab293220c14900c6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Wed, 29 Jul 2026 00:17:19 -0700 Subject: [PATCH 2/2] fix(db): carry locked through the reconcile and start suffixes at (1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings, both real. `locked` was hardcoded false on the workflow insert. `workflow_folder` has a `locked` column and the app still enforces the lock feature against `folder.locked`, so a stranded locked folder would have come back unlocked. 0272 carries `f.locked` through for exactly this reason; this now matches it. `workspace_file_folders` has no such column — file folders have never been lockable — so the file pass still writes false. Name dedup started at ' (2)', skipping a free ' (1)'. 0272 enumerates `generate_series(1, 10000)` and the app's own dedup produces "New folder (1)", "New folder (2)", ..., so starting at (2) was inconsistent with both. Fixtures extended to cover both: a stranded locked folder must arrive locked, a stranded unlocked one must stay unlocked (proving the value is copied rather than blanket-set), and a collision with a free ' (1)' must take it. Still idempotent across three consecutive runs. The single row stranded in production is an archived, unlocked canary folder, so neither bug would have bitten on today's run — but this file is meant to be re-runnable and ships to self-hosted rolling deploys. --- ...75_folder_cutover_post_drain_reconcile.sql | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql b/packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql index fedc4f9e36f..e1965efc49c 100644 --- a/packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql +++ b/packages/db/migrations/0275_folder_cutover_post_drain_reconcile.sql @@ -22,10 +22,17 @@ -- `folder` are touched; every row already mirrored is left exactly as it is. -- -- Names are deduplicated against the FIRST FREE suffix rather than a row number, because the --- target name may already be taken by a row this block is not inserting. Only ACTIVE rows are --- deduplicated — the partial unique index ignores soft-deleted rows, so an archived folder +-- target name may already be taken by a row this block is not inserting. Suffixes start at +-- ' (1)' to match 0272 and the app's own dedup, which both produce "New folder (1)", +-- "New folder (2)", ... — starting at (2) would skip a legitimately free name. Only ACTIVE rows +-- are deduplicated: the partial unique index ignores soft-deleted rows, so an archived folder -- keeps its original name and reappears under it if restored. -- +-- `locked` is carried over from `workflow_folder` rather than defaulted, exactly as 0272 does. +-- The lock feature is still enforced against `folder.locked`, so defaulting a stranded locked +-- folder to false would silently unlock it. `workspace_file_folders` has no such column — file +-- folders have never been lockable — so the file pass writes false. +-- -- Rows are inserted in depth order so a parent always lands before its children: the -- self-referencing FK `folder_parent_id_folder_id_fk` is checked immediately, and the -- `folder_parent_resource_type_match` trigger reads the parent row. A child inserted before @@ -64,7 +71,7 @@ BEGIN FROM "workflow_folder" s JOIN tree t ON t.id = s."parent_id" ) - SELECT s."id", s."name", s."user_id", s."workspace_id", s."parent_id", + SELECT s."id", s."name", s."user_id", s."workspace_id", s."parent_id", s."locked", s."sort_order", s."created_at", s."updated_at", s."archived_at" AS deleted_at FROM "workflow_folder" s JOIN tree t ON t.id = s."id" @@ -73,7 +80,7 @@ BEGIN LOOP candidate := rec."name"; IF rec.deleted_at IS NULL THEN - suffix := 1; + suffix := 0; WHILE EXISTS ( SELECT 1 FROM "folder" f WHERE f."workspace_id" = rec."workspace_id" @@ -89,7 +96,7 @@ BEGIN INSERT INTO "folder" (id, resource_type, name, user_id, workspace_id, parent_id, locked, sort_order, created_at, updated_at, deleted_at) VALUES (rec."id", 'workflow', candidate, rec."user_id", rec."workspace_id", rec."parent_id", - false, rec."sort_order", rec."created_at", rec."updated_at", rec.deleted_at) + rec."locked", rec."sort_order", rec."created_at", rec."updated_at", rec.deleted_at) ON CONFLICT (id) DO NOTHING; END LOOP; @@ -114,7 +121,7 @@ BEGIN LOOP candidate := rec."name"; IF rec."deleted_at" IS NULL THEN - suffix := 1; + suffix := 0; WHILE EXISTS ( SELECT 1 FROM "folder" f WHERE f."workspace_id" = rec."workspace_id"