feat(db): store DOI, raw read and target gene identifiers on the rows that use them

.claude/rules/database.md

@@ -1,5 +1,6 @@
 ---
 paths:
+  - "src/mavedb/db/**"
   - "src/mavedb/models/**"
   - "src/mavedb/lib/**"
   - "src/mavedb/routers/**"
@@ -12,3 +13,4 @@ paths:
 - Write new queries in SQLAlchemy 2.0 `select()` style. Older code still uses `db.query()`; don't copy it.
 - ValidTime tables never return only live rows by default. Filter with `Model.current` or `Model.live_at(...)`.
 - Replace ValidTime rows with `supersede_with` or `supersede_live_where`, never a separate retire-then-insert, so the handoff has no gap.
+- Mixins shared by models live in `models/mixins/<concept>.py`, such as `ValidTime`. `db/` holds what every table builds on: `Base`, `MaterializedView` and sessions. Each top-level `models/` module defines a table.

alembic/versions/2dfdc4a88870_store_identifiers_on_owner_rows.py

@@ -0,0 +1,197 @@
+"""store DOI, raw read and target gene identifiers on the rows that use them
+
+Revision ID: 2dfdc4a88870
+Revises: 6987f253e02a
+Create Date: 2026-10-21
+
+DOI, raw read (SRA), Ensembl, RefSeq and UniProt identifiers each lived in a shared table that records linked
+to by id. Every column but the identifier was a constant, always null, or derivable from it, so the identifier
+moves onto the row that uses it (#944):
+
+- The six DOI and raw read link tables hold the identifier string, keyed by ``(<parent>_id, identifier)``.
+  DOIs are stored bare and upper-cased, as the api now stores new ones: ``doi:`` and resolver prefixes are
+  stripped, and upper case is the form the DOI registry stores, since DOIs are case-insensitive and a record
+  shouldn't cite one twice.
+- ``ensembl_offsets``, ``refseq_offsets`` and ``uniprot_offsets`` merge into
+  ``target_gene_external_identifiers``, keyed by ``(target_gene_id, db_name)``: a target gene has at most one
+  identifier per database.
+- The five shared identifier tables are dropped, with any rows no record linked to. Urls are computed when
+  serializing.
+
+Under row-level security (#808), these become owned rows, visible exactly when their record is.
+
+The tables are small (a few thousand rows in all), so the rewrites hold their locks briefly.
+
+The downgrade rebuilds the shared tables from the link rows, so the unlinked rows don't come back. Ids are
+reassigned, the dates are the day of the downgrade, and urls are rebuilt the way the api used to build them, so
+gene identifier urls come back null.
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "2dfdc4a88870"
+down_revision = "6987f253e02a"
+branch_labels = None
+depends_on = None
+
+# The api accepted DOIs with a ``doi:`` or resolver prefix before #944 and stored them as given. This strips the
+# prefixes idutils.normalize_doi strips, which is what the api applies to new DOIs.
+DOI_NORMALIZE = r"upper(regexp_replace({}, '^(doi:\s*|(https?://)?(dx\.)?doi\.org/)', '', 'i'))"
+
+# (link table, parent column, shared table, shared id column)
+DOI_LINKS = [
+    ("scoreset_doi_identifiers", "scoreset_id", "doi_identifiers", "doi_identifier_id"),
+    ("experiment_doi_identifiers", "experiment_id", "doi_identifiers", "doi_identifier_id"),
+    ("experiment_set_doi_identifiers", "experiment_set_id", "doi_identifiers", "doi_identifier_id"),
+]
+RAW_READ_LINKS = [
+    ("scoreset_sra_identifiers", "scoreset_id", "sra_identifiers", "sra_identifier_id"),
+    ("experiment_sra_identifiers", "experiment_id", "sra_identifiers", "sra_identifier_id"),
+    ("experiment_set_sra_identifiers", "experiment_set_id", "sra_identifiers", "sra_identifier_id"),
+]
+
+# (db_name, offset table, identifier table, whether the identifier table has reference_html)
+GENE_IDENTIFIERS = [
+    ("Ensembl", "ensembl_offsets", "ensembl_identifiers", False),
+    ("RefSeq", "refseq_offsets", "refseq_identifiers", True),
+    ("UniProt", "uniprot_offsets", "uniprot_identifiers", True),
+]
+
+
+def _fold_link_table(link: str, parent: str, shared: str, shared_id: str, normalize: str) -> None:
+    op.execute(f"ALTER TABLE {link} ADD COLUMN identifier character varying")
+    op.execute(
+        f"UPDATE {link} l SET identifier = {normalize.format('s.identifier')} FROM {shared} s WHERE s.id = l.{shared_id}"
+    )
+
+    # Two shared rows differing only in case or prefix would collide once normalized. There are none today.
+    op.execute(
+        f"DELETE FROM {link} a USING {link} b "
+        f"WHERE a.{parent} = b.{parent} AND a.identifier = b.identifier AND a.ctid > b.ctid"
+    )
+
+    op.execute(f"ALTER TABLE {link} DROP CONSTRAINT {link}_pkey")
+    op.execute(f"ALTER TABLE {link} DROP COLUMN {shared_id}")
+    op.execute(f"ALTER TABLE {link} ALTER COLUMN identifier SET NOT NULL")
+    op.execute(f"ALTER TABLE {link} ADD CONSTRAINT {link}_pkey PRIMARY KEY ({parent}, identifier)")
+
+
+def _create_shared_table(shared: str, with_reference_html: bool = False) -> None:
+    reference_html = "reference_html character varying," if with_reference_html else ""
+    op.execute(
+        f"""
+        CREATE TABLE {shared} (
+            id serial NOT NULL,
+            identifier character varying NOT NULL,
+            db_name character varying NOT NULL,
+            db_version character varying,
+            url character varying,
+            {reference_html}
+            creation_date date NOT NULL,
+            modification_date date NOT NULL,
+            CONSTRAINT {shared}_pkey PRIMARY KEY (id)
+        )
+        """
+    )
+
+
+def _unfold_link_table(link: str, parent: str, shared: str, shared_id: str) -> None:
+    op.execute(f"ALTER TABLE {link} ADD COLUMN {shared_id} integer")
+    op.execute(f"UPDATE {link} l SET {shared_id} = s.id FROM {shared} s WHERE s.identifier = l.identifier")
+    op.execute(f"ALTER TABLE {link} DROP CONSTRAINT {link}_pkey")
+    op.execute(f"ALTER TABLE {link} DROP COLUMN identifier")
+    op.execute(f"ALTER TABLE {link} ALTER COLUMN {shared_id} SET NOT NULL")
+    op.execute(f"ALTER TABLE {link} ADD CONSTRAINT {link}_pkey PRIMARY KEY ({parent}, {shared_id})")
+    op.execute(
+        f"ALTER TABLE {link} ADD CONSTRAINT {link}_{shared_id}_fkey FOREIGN KEY ({shared_id}) REFERENCES {shared}(id)"
+    )
+
+
+def upgrade() -> None:
+    for link, parent, shared, shared_id in DOI_LINKS:
+        _fold_link_table(link, parent, shared, shared_id, normalize=DOI_NORMALIZE)
+    for link, parent, shared, shared_id in RAW_READ_LINKS:
+        _fold_link_table(link, parent, shared, shared_id, normalize="{}")
+
+    op.execute(
+        """
+        CREATE TABLE target_gene_external_identifiers (
+            target_gene_id integer NOT NULL,
+            db_name character varying NOT NULL,
+            identifier character varying NOT NULL,
+            "offset" integer NOT NULL,
+            CONSTRAINT target_gene_external_identifiers_pkey PRIMARY KEY (target_gene_id, db_name),
+            CONSTRAINT target_gene_external_identifiers_target_gene_id_fkey
+                FOREIGN KEY (target_gene_id) REFERENCES target_genes(id) ON DELETE CASCADE,
+            CONSTRAINT ck_target_gene_external_identifiers_db_name_valid
+                CHECK (db_name IN ('Ensembl', 'RefSeq', 'UniProt'))
+        )
+        """
+    )
+    for db_name, offsets, identifiers, _ in GENE_IDENTIFIERS:
+        op.execute(
+            f"""
+            INSERT INTO target_gene_external_identifiers (target_gene_id, db_name, identifier, "offset")
+            SELECT o.target_gene_id, '{db_name}', i.identifier, o."offset"
+              FROM {offsets} o JOIN {identifiers} i ON i.id = o.identifier_id
+            """
+        )
+        op.execute(f"DROP TABLE {offsets}")
+        op.execute(f"DROP TABLE {identifiers}")
+
+    op.execute("DROP TABLE doi_identifiers")
+    op.execute("DROP TABLE sra_identifiers")
+
+
+def downgrade() -> None:
+    for shared, links, db_name, url_prefix in [
+        ("doi_identifiers", DOI_LINKS, "DOI", "https://doi.org/"),
+        ("sra_identifiers", RAW_READ_LINKS, "SRA", "http://www.ebi.ac.uk/ena/data/view/"),
+    ]:
+        _create_shared_table(shared)
+        every_identifier = " UNION ".join(f"SELECT identifier FROM {link}" for link, *_ in links)
+        op.execute(
+            f"""
+            INSERT INTO {shared} (identifier, db_name, url, creation_date, modification_date)
+            SELECT identifier, '{db_name}', '{url_prefix}' || identifier, current_date, current_date
+              FROM ({every_identifier}) AS every_identifier
+             ORDER BY identifier
+            """
+        )
+        for link, parent, _, shared_id in links:
+            _unfold_link_table(link, parent, shared, shared_id)
+
+    for db_name, offsets, identifiers, with_reference_html in GENE_IDENTIFIERS:
+        _create_shared_table(identifiers, with_reference_html)
+        op.execute(
+            f"""
+            INSERT INTO {identifiers} (identifier, db_name, creation_date, modification_date)
+            SELECT DISTINCT identifier, db_name, current_date, current_date
+              FROM target_gene_external_identifiers WHERE db_name = '{db_name}'
+             ORDER BY identifier
+            """
+        )
+        op.execute(
+            f"""
+            CREATE TABLE {offsets} (
+                identifier_id integer NOT NULL,
+                target_gene_id integer NOT NULL,
+                "offset" integer NOT NULL,
+                CONSTRAINT {offsets}_pkey PRIMARY KEY (identifier_id, target_gene_id),
+                CONSTRAINT {offsets}_identifier_id_fkey FOREIGN KEY (identifier_id) REFERENCES {identifiers}(id),
+                CONSTRAINT {offsets}_target_gene_id_fkey FOREIGN KEY (target_gene_id) REFERENCES target_genes(id)
+            )
+            """
+        )
+        op.execute(
+            f"""
+            INSERT INTO {offsets} (identifier_id, target_gene_id, "offset")
+            SELECT i.id, t.target_gene_id, t."offset"
+              FROM target_gene_external_identifiers t
+              JOIN {identifiers} i ON i.identifier = t.identifier
+             WHERE t.db_name = '{db_name}'
+            """
+        )
+
+    op.execute("DROP TABLE target_gene_external_identifiers")

src/mavedb/lib/external_identifier_urls.py

@@ -0,0 +1,47 @@
+"""Links to the external databases that records cite through :class:`ExternalIdentifier` models.
+
+MaveDB has no other code for these databases, so their link rules live together here. A source MaveDB also
+talks to, such as gnomAD or ClinGen, keeps its link with that code. This module imports nothing from mavedb, so
+models can use it.
+"""
+
+import re
+
+import idutils
+
+# NCBI keeps RefSeq proteins (these prefixes) in its `protein` database and everything else in `nuccore`, its
+# nucleotide database. A link to the wrong one finds nothing.
+REFSEQ_PROTEIN_ACCESSION = re.compile(r"^(AP|NP|WP|XP|YP)_")
+
+
+def doi_url(identifier: str) -> str:
+    """The resolver link for a DOI."""
+    return f"https://doi.org/{identifier}"
+
+
+def raw_read_url(identifier: str) -> str:
+    """The page for a raw read accession, chosen by its accession type (#445)."""
+    if idutils.is_bioproject(identifier):
+        return f"https://www.ncbi.nlm.nih.gov/bioproject/{identifier}"
+    if idutils.is_geo(identifier):
+        return f"https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={identifier}"
+    if idutils.is_arrayexpress_experiment(identifier):
+        return f"https://www.ebi.ac.uk/biostudies/arrayexpress/studies/{identifier}"
+    if idutils.is_arrayexpress_array(identifier):
+        return f"https://www.ebi.ac.uk/biostudies/arrayexpress/arrays/{identifier}"
+
+    # SRA accessions (SRR, ERR, DRR and their study and experiment forms) are shared across the INSDC archives,
+    # and ENA's browser resolves all of them. It's also where every raw read url pointed before #445, so
+    # accessions that match none of the types above keep working.
+    return f"https://www.ebi.ac.uk/ena/browser/view/{identifier}"
+
+
+def external_gene_identifier_url(db_name: str, identifier: str) -> str:
+    """The page for a target gene's identifier in Ensembl, RefSeq or UniProt."""
+    if db_name == "Ensembl":
+        return f"https://www.ensembl.org/id/{identifier}"
+    if db_name == "UniProt":
+        return f"https://www.uniprot.org/uniprotkb/{identifier}"
+
+    database = "protein" if REFSEQ_PROTEIN_ACCESSION.match(identifier) else "nuccore"
+    return f"https://www.ncbi.nlm.nih.gov/{database}/{identifier}"

src/mavedb/lib/identifier_search.py

@@ -0,0 +1,103 @@
+"""Search for the external identifiers that records cite, limited to records the caller may read.
+
+This is kept apart from :mod:`mavedb.lib.identifiers` because it needs the permission checks, which import the
+web framework, and view models import that module.
+"""
+
+from typing import Any, Sequence, Union
+
+from sqlalchemy import Select, func, select
+from sqlalchemy.orm import Session
+
+from mavedb.lib.permissions.experiment import ExperimentViewer
+from mavedb.lib.permissions.experiment_set import ExperimentSetViewer
+from mavedb.lib.permissions.principal import Principal
+from mavedb.lib.permissions.score_set import ScoreSetViewer
+from mavedb.lib.permissions.viewer import Viewer
+from mavedb.models.doi_identifier import ExperimentDoiIdentifier, ExperimentSetDoiIdentifier, ScoreSetDoiIdentifier
+from mavedb.models.experiment import Experiment
+from mavedb.models.experiment_set import ExperimentSet
+from mavedb.models.mixins.external_identifier import ExternalIdentifier
+from mavedb.models.raw_read_identifier import (
+    ExperimentRawReadIdentifier,
+    ExperimentSetRawReadIdentifier,
+    ScoreSetRawReadIdentifier,
+)
+from mavedb.models.score_set import ScoreSet
+
+IDENTIFIER_SEARCH_LIMIT = 50
+
+# A table to search for identifiers: its link model, the viewer for the record that governs each row's
+# visibility, and a query selecting ``(link row, record)``.
+IdentifierSource = tuple[type[ExternalIdentifier], type[Viewer[Any]], Select]
+
+
+def search_identifiers_on_readable_records(
+    db: Session,
+    principal: Principal,
+    text: str,
+    sources: Sequence[IdentifierSource],
+) -> list[ExternalIdentifier]:
+    """Find identifiers containing ``text`` that are attached to records the caller may read.
+
+    Identifiers are stored on the rows that link them to their records, so one identifier can appear on many
+    rows. This returns one row per distinct identifier, sorted by identifier.
+
+    Args:
+        db: An active database session.
+        principal: The caller.
+        text: The text to search for, matched case-insensitively anywhere in the identifier.
+        sources: The tables to search.
+
+    Returns:
+        Up to ``IDENTIFIER_SEARCH_LIMIT`` link rows, one per distinct identifier.
+    """
+    # Read access to a record is decided in Python, so the filter runs before the limit. These tables hold a
+    # few hundred rows. Once row-level security enforces reads (#808), the database does this filtering.
+    found: dict[str, ExternalIdentifier] = {}
+    for link, viewer_class, query in sources:
+        viewer = principal.viewer_for(viewer_class)
+        rows = db.execute(query.where(func.lower(link.identifier).contains(text.lower(), autoescape=True))).all()
+        for link_row, record in rows:
+            if link_row.identifier in found:
+                continue
+            if viewer.may_read(record):
+                found[link_row.identifier] = link_row
+
+    return [found[identifier] for identifier in sorted(found)][:IDENTIFIER_SEARCH_LIMIT]
+
+
+def identifier_sources_on_records(
+    score_set_link: Union[type[ScoreSetDoiIdentifier], type[ScoreSetRawReadIdentifier]],
+    experiment_link: Union[type[ExperimentDoiIdentifier], type[ExperimentRawReadIdentifier]],
+    experiment_set_link: Union[type[ExperimentSetDoiIdentifier], type[ExperimentSetRawReadIdentifier]],
+) -> list[IdentifierSource]:
+    """The sources for an identifier that score sets, experiments and experiment sets each cite.
+
+    Args:
+        score_set_link: The model linking the identifier to score sets.
+        experiment_link: The model linking the identifier to experiments.
+        experiment_set_link: The model linking the identifier to experiment sets.
+
+    Returns:
+        A source for each record type, for :func:`search_identifiers_on_readable_records`.
+    """
+    return [
+        (
+            score_set_link,
+            ScoreSetViewer,
+            select(score_set_link, ScoreSet).join(ScoreSet, ScoreSet.id == score_set_link.score_set_id),
+        ),
+        (
+            experiment_link,
+            ExperimentViewer,
+            select(experiment_link, Experiment).join(Experiment, Experiment.id == experiment_link.experiment_id),
+        ),
+        (
+            experiment_set_link,
+            ExperimentSetViewer,
+            select(experiment_set_link, ExperimentSet).join(
+                ExperimentSet, ExperimentSet.id == experiment_set_link.experiment_set_id
+            ),
+        ),
+    ]