Implement lazy sub-row fetching for non-paginateSubRows grouped mode

Only fetch sub-rows for expanded groups instead of all visible groups.
Collapsed groups get empty subRows with subRowCount so expander arrows show.

Changes:
- DuckDBBackend.js: buildGroupLevel() adds COUNT(*) to GROUP BY, skips
  sub-row fetch for collapsed groups, sets __state.subRowCount
- Reactable.js: always pass expanded to DuckDB query when groupBy active,
  fix eslint missing deps warning (canSkipInitialDuckDBQuery, defaultSorted,
  paginateSubRows), extend subRowCount placeholder to all backend modes
- backend-duckdb.R: duckdbGroupedQuery() accepts expanded/parentId, adds
  COUNT(*) AS _sub_row_count, skips sub-row fetch for collapsed groups
- backend-df.R: dfGroupBy() accepts expanded/parentId, trims sub-rows for
  collapsed groups after computing subRowCount

expanded=NULL/undefined means all groups expanded (fetch all sub-rows),
matching pre-existing behavior.

Fix nondeterministic unique aggregator: add ORDER BY 1 to STRING_AGG(DISTINCT ...)
so unique values are sorted alphabetically. Affects both DuckDB WASM client
(DuckDBBackend.js) and DuckDB R server (duckdb-sql.R).

Document known issues in implementation plan:
- Select-all has no fallback for custom backends
- Refactor selectAll into separate S3 generic

Tests: 4 JS tests, 4 df R tests, 3 DuckDB R tests. All pass.

Author: glin

R/backend-df.R

@@ -60,7 +60,7 @@ reactableServerData.reactable_backendDf <- function(
 
   # Grouping and aggregation
   if (length(groupBy) > 0) {
-    data <- dfGroupBy(data, groupBy, columns)
+    data <- dfGroupBy(data, groupBy, columns, expanded = expanded)
   }
 
   # Pagination
@@ -106,7 +106,7 @@ dfSortBy <- function(df, by) {
   df
 }
 
-dfGroupBy <- function(df, by, columns = NULL, depth = 0) {
+dfGroupBy <- function(df, by, columns = NULL, depth = 0, expanded = NULL, parentId = NULL) {
   by <- unlist(by)
   if (length(by) == depth) {
     return(df)
@@ -166,28 +166,46 @@ dfGroupBy <- function(df, by, columns = NULL, depth = 0) {
     }), recursive = FALSE)
   }
 
-  df[[".subRows"]] <- lapply(values, function(x) {
+  # Compute row state IDs before building .subRows (needed for expanded check and parentId)
+  rowIds <- unname(vapply(values, function(x) {
+    value <- if (is.list(x)) toJSON(x) else as.character(x)
+    sprintf("%s:%s", groupedColumnId, value)
+  }, character(1)))
+
+  df[[".subRows"]] <- lapply(seq_along(values), function(j) {
+    x <- values[[j]]
     value <- if (is.list(x)) toJSON(x) else as.character(x)
     subGroup <- groups[[value]]
-    dfGroupBy(subGroup, by, columns = columns, depth = depth + 1)
+    childParentId <- if (!is.null(parentId)) paste0(parentId, ".", rowIds[j]) else rowIds[j]
+    dfGroupBy(subGroup, by, columns = columns, depth = depth + 1,
+              expanded = expanded, parentId = childParentId)
   })
 
   # Add row state for grouped rows. This includes:
 
   # - id: unique identifier for the row (format: "columnId:value")
   # - grouped: TRUE to mark this as a grouped row
-  # - subRowCount: count of sub rows (for paginateSubRows)
-  rowIds <- unname(vapply(values, function(x) {
-    value <- if (is.list(x)) toJSON(x) else as.character(x)
-    sprintf("%s:%s", groupedColumnId, value)
-  }, character(1)))
+  # - subRowCount: count of sub rows (for paginateSubRows and lazy sub-row fetching)
   subRowCounts <- vapply(df[[".subRows"]], nrow, integer(1))
   df[["__state"]] <- dataFrame(
     id = rowIds,
     grouped = rep(TRUE, length(rowIds)),
     subRowCount = subRowCounts
   )
 
+  # Trim sub-rows for collapsed groups (lazy sub-row fetching).
+  # When expanded is provided, only expanded groups keep their sub-rows.
+  # Collapsed groups get empty data frames; subRowCount is preserved in __state
+  # so the expander arrow still shows.
+  if (!is.null(expanded)) {
+    for (i in seq_along(rowIds)) {
+      fullRowId <- if (!is.null(parentId)) paste0(parentId, ".", rowIds[i]) else rowIds[i]
+      if (!isTRUE(expanded[[fullRowId]])) {
+        df[[".subRows"]][[i]] <- df[[".subRows"]][[i]][0, , drop = FALSE]
+      }
+    }
+  }
+
   df
 }
 

R/backend-duckdb.R

@@ -153,7 +153,7 @@ reactableServerData.reactable_backendDuckdb <- function(
                                    pageIndex, pageSize, groupBy, expanded))
     }
     return(duckdbGroupedQuery(con, cols, filters, searchValue, sortBy,
-                              pageIndex, pageSize, groupBy))
+                              pageIndex, pageSize, groupBy, expanded))
   }
 
   query <- buildDuckdbQuery(
@@ -187,15 +187,16 @@ reactableServerData.reactable_backendDuckdb <- function(
 # Execute a grouped query: GROUP BY for top level, then sub-rows for each group.
 # Returns resolvedData with nested .subRows matching the server-df format.
 duckdbGroupedQuery <- function(con, columns, filters, searchValue, sortBy,
-                                pageIndex, pageSize, groupBy,
-                                depth = 0, parentFilters = list(clauses = character(0), params = list())) {
+                                pageIndex, pageSize, groupBy, expanded = NULL,
+                                depth = 0, parentFilters = list(clauses = character(0), params = list()),
+                                parentId = NULL) {
   groupCol <- groupBy[[depth + 1]]
   groupedCols <- groupBy[seq_len(depth + 1)]
   baseWhere <- buildDuckdbWhere(columns, filters, searchValue)
 
-  # Build SELECT with group column + SQL aggregates
+  # Build SELECT with group column + COUNT(*) for sub-row counts + SQL aggregates
   escapedGroupCol <- duckdbQuoteIdentifier(groupCol)
-  selectParts <- escapedGroupCol
+  selectParts <- c(escapedGroupCol, "COUNT(*) AS _sub_row_count")
   postComputeAggs <- list()
 
   for (col in columns) {
@@ -249,29 +250,51 @@ duckdbGroupedQuery <- function(con, columns, filters, searchValue, sortBy,
     return(groupData)
   }
 
-  # Fetch sub-rows for each group
+  # Extract sub-row counts and remove the helper column
+  subRowCounts <- groupData[["_sub_row_count"]]
+  groupData[["_sub_row_count"]] <- NULL
+
+  # Classify groups as expanded or collapsed.
+  # expanded=NULL means all groups expanded (no lazy fetching, backward compat).
+  # expanded=list() means nothing expanded (all collapsed).
   groupValues <- groupData[[groupCol]]
+  stateIds <- vapply(groupValues, function(val) paste0(groupCol, ":", val), character(1))
+  rowIds <- if (!is.null(parentId)) paste0(parentId, ".", stateIds) else stateIds
+  if (is.null(expanded)) {
+    isExpanded <- rep(TRUE, length(groupValues))
+  } else {
+    isExpanded <- vapply(rowIds, function(id) isTRUE(expanded[[id]]), logical(1))
+  }
+
+  # Fetch sub-rows only for expanded groups
   subRowsList <- vector("list", length(groupValues))
 
   for (i in seq_along(groupValues)) {
+    if (!isExpanded[[i]]) {
+      # Collapsed group: empty sub-rows (subRowCount set in __state below)
+      subRowsList[[i]] <- data.frame()
+      next
+    }
+
     childFilters <- list(
       clauses = c(parentFilters$clauses, paste0(escapedGroupCol, " = ?")),
       params = c(parentFilters$params, list(groupValues[[i]]))
     )
 
     if (depth + 1 < length(groupBy)) {
-      # More grouping levels — recurse
+      # More grouping levels - recurse only for expanded groups
       subResult <- duckdbGroupedQuery(con, columns, filters, searchValue, sortBy,
                                        pageIndex = 0, pageSize = .Machine$integer.max,
-                                       groupBy, depth = depth + 1,
-                                       parentFilters = childFilters)
+                                       groupBy, expanded = expanded, depth = depth + 1,
+                                       parentFilters = childFilters,
+                                       parentId = rowIds[[i]])
       if (inherits(subResult, "reactable_resolvedData")) {
         subRowsList[[i]] <- subResult$data
       } else {
         subRowsList[[i]] <- subResult
       }
     } else {
-      # Leaf level -- fetch individual rows
+      # Leaf level - fetch individual rows
       subQuery <- buildDuckdbSubRowSql("reactable_data", baseWhere, childFilters, sortBy)
       subRows <- DBI::dbGetQuery(con, subQuery$sql, params = subQuery$params)
       # Extract _reactable_rowid into __state for stable row identification
@@ -297,10 +320,12 @@ duckdbGroupedQuery <- function(con, columns, filters, searchValue, sortBy,
 
   groupData[[".subRows"]] <- subRowsList
 
-  # Add __state with group ID for group header rows
+  # Add __state with group ID for group header rows.
+  # Collapsed groups include subRowCount so the expander arrow shows.
   groupData[["__state"]] <- dataFrame(
-    id = vapply(groupValues, function(val) paste0(groupCol, ":", val), character(1)),
-    grouped = rep(TRUE, length(groupValues))
+    id = unname(stateIds),
+    grouped = rep(TRUE, length(stateIds)),
+    subRowCount = ifelse(isExpanded, NA_integer_, as.integer(subRowCounts))
   )
 
   if (depth == 0) {

R/duckdb-sql.R

@@ -202,7 +202,7 @@ duckdbAggregateSQL <- function(aggregate, columnId) {
     "min" = paste0("MIN(", col, ")"),
     "median" = paste0("MEDIAN(", col, ")"),
     "count" = paste0("COUNT(", col, ")"),
-    "unique" = paste0("STRING_AGG(DISTINCT CAST(", col, " AS VARCHAR), ', ')"),
+    "unique" = paste0("STRING_AGG(DISTINCT CAST(", col, " AS VARCHAR), ', ' ORDER BY 1)"),
     NULL  # frequency and unknown aggregates are computed from sub-rows
   )
 }

R/shiny.R

@@ -363,7 +363,8 @@ getReactableState <- function(outputId, name = NULL, session = NULL) {
 #' @param groupBy The current grouped columns. `NULL` if empty.
 #' @param pagination Whether pagination is enabled, `TRUE` or `FALSE`.
 #' @param paginateSubRows Whether sub rows are paginated, `TRUE` or `FALSE`.
-#' @param selectAll Whether a select-all operation is being requested, `TRUE` or `NULL`.
+#' @param selectAll Whether a select-all operation is being requested. `TRUE` if the
+#'   user clicked the select-all checkbox, or `NULL` for normal data requests.
 #' @param expanded The current expanded rows.
 #' @param ... Additional arguments passed to the S3 method.
 #' @return

design/duckdb-wasm-engine/implementation-plan.md

@@ -833,6 +833,33 @@ integer vector via `setdiff(seq_len(rowCount), deselected)`. The payload is self
    selection state. This matches client-side semantics where select-all only applies to
    the currently filtered rows.
 
+3. **~~`unique` aggregator is nondeterministic:~~** Fixed. Added `ORDER BY 1` inside the
+   `STRING_AGG` in both DuckDB WASM client (`DuckDBBackend.js`) and DuckDB R server
+   (`duckdb-sql.R`). Values are now sorted alphabetically, making the output deterministic.
+   The `frequency` aggregator was also checked: it's computed from sub-rows using
+   `Object.entries(counts)` in JS and `table()` in R. The R version sorts alphabetically
+   (via `table()`); the JS version uses insertion order. Both are stable within each
+   backend, so no fix needed (minor cross-backend ordering difference is acceptable).
+
+4. **Select-all has no fallback for custom backends:** The `toggleAllRowsSelected` override
+   in `Reactable.js` queries the backend for all matching row IDs (`selectAll: true`). If a
+   custom backend doesn't implement `selectAll` (returns empty `rowIds` or errors), select-all
+   effectively selects zero rows instead of falling back to current-page rows. Fix: save the
+   original `toggleAllRowsSelected` before overriding it, and if `getMatchingRowIds()` returns
+   an empty array, call through to the original (which enumerates `nonGroupedRowsById`, i.e.,
+   current-page rows). This makes the documented behavior in `?reactableServerData` accurate:
+   "selection will only work for rows on the current page" when the backend doesn't support it.
+
+5. **Refactor `selectAll` out of `reactableServerData`:** The `selectAll` parameter overloads
+   `reactableServerData()` to serve two unrelated purposes (return page data vs. return all
+   matching row IDs) with a polymorphic return type. Refactor into a separate S3 generic,
+   e.g. `reactableServerSelectAll(x, data, columns, filters, searchValue, ...)`. Provide a
+   default method that calls `reactableServerData()` without pagination and extracts row IDs
+   from `__state$id`, so custom backends get cross-page select-all for free without
+   implementing anything extra. Keep the existing overloaded approach working for backward
+   compatibility (if a backend handles `selectAll` in `reactableServerData`, use that;
+   otherwise dispatch to the new generic).
+
 #### 9E: Server-side expansion (lazy sub-row fetching)
 
 **Goal:** Only fetch sub-rows for expanded groups, not all groups on the page. Currently, in
@@ -901,32 +928,32 @@ needed for that path.
 
 **Steps:**
 
-- [ ] **9E.1** **Reactable.js:** Always pass `expanded: state.expanded` (not just when
+- [x] **9E.1** **Reactable.js:** Always pass `expanded: state.expanded` (not just when
       `paginateSubRows`) in the DuckDB query effect when `groupBy` is active. Add
-      `state.expanded` to the dependency array for grouped DuckDB queries. Extend the
-      `subRowCount` placeholder pattern to apply in `(useDuckDB || useServerData) && !paginateSubRows`
-      mode (currently only `paginateSubRows`).
-- [ ] **9E.2** **Reactable.js (server data):** The V8 server useEffect already sends `expanded`
-      and has it in deps. For non-V8 server backends (df/duckdb-server), the expanded state
-      needs to reach the backend. Verify that the server POST includes `expanded` and that the
-      R `reactableServerData` methods pass it through to the grouping functions.
-- [ ] **9E.3** **DuckDB WASM (`DuckDBBackend.js`):** Update `queryGrouped()` to accept
-      `expanded` param. At leaf level, split groups into expanded/collapsed. For expanded groups,
-      fetch sub-rows via `IN()` as before. For collapsed groups, skip the sub-row query and set
-      `.subRows = []` with `__state.subRowCount` from `COUNT(*)` (needs to be added to
-      `buildGroupLevel()` SELECT; see `_sub_count` in `buildPaginatedGroupTree` for reference).
-      For multi-level, only recurse into expanded groups.
-- [ ] **9E.4** **DuckDB R server (`backend-duckdb.R`):** Update `duckdbGroupedQuery()` to
-      accept and use `expanded`. Add `COUNT(*) AS _sub_row_count` to the GROUP BY select if not
-      already present. Skip sub-row fetch for collapsed groups. Set `__state$subRowCount`.
-      Pass `expanded` from `reactableServerData` to `duckdbGroupedQuery()`.
-- [ ] **9E.5** **df backend (`backend-df.R`):** Update `dfGroupBy()` or add post-processing in
-      `reactableServerData` to trim sub-rows for collapsed groups. Set `__state$subRowCount`
-      for collapsed groups. Pass `expanded` through from `reactableServerData`.
-- [ ] **9E.6** **Tests:** JS tests: grouped table with DuckDB, expand group triggers re-query
-      and shows sub-rows, collapse group triggers re-query and hides sub-rows. Verify collapsed
-      groups show expander arrow (canExpand is true). R tests: verify `__state$subRowCount` in
-      responses, verify sub-rows only returned for expanded groups.
+      `state.expanded` to the dependency array for grouped DuckDB queries. Removed the
+      `paginateSubRows` guard on the `subRowCount` placeholder pattern (the inner
+      `subRowCount != null` check is sufficient since we're inside `manualPagination`).
+- [x] **9E.2** **Reactable.js (server data):** Verified: the V8 server useEffect already
+      sends `expanded: state.expanded` and has it in deps. No changes needed; the server
+      POST already includes `expanded` for all backends.
+- [x] **9E.3** **DuckDB WASM (`DuckDBBackend.js`):** `queryGrouped()` accepts `expanded`.
+      `buildGroupLevel()` adds `COUNT(*) AS _sub_count` to GROUP BY SELECT, classifies
+      groups as expanded/collapsed, only fetches sub-rows (IN() query) for expanded groups.
+      Collapsed groups get `.subRows = []` with `__state.subRowCount`. Multi-level only
+      recurses into expanded groups. `expanded=undefined` means all expanded (backward compat);
+      `expanded={}` means all collapsed.
+- [x] **9E.4** **DuckDB R server (`backend-duckdb.R`):** `duckdbGroupedQuery()` accepts
+      `expanded` and `parentId`. Adds `COUNT(*) AS _sub_row_count` to GROUP BY SELECT.
+      Skips sub-row fetch for collapsed groups. Sets `__state$subRowCount` (NA for expanded,
+      integer for collapsed). `expanded=NULL` is backward compat (all expanded).
+- [x] **9E.5** **df backend (`backend-df.R`):** `dfGroupBy()` accepts `expanded` and
+      `parentId`. After building full sub-rows and computing `subRowCount`, trims sub-rows
+      for collapsed groups to empty data frames. `expanded=NULL` is backward compat.
+      Passes `expanded` and `parentId` through recursion for multi-level groupBy.
+- [x] **9E.6** **Tests:** 4 new JS tests (collapsed groups, expanded/collapsed mix,
+      multi-level collapsed, multi-level expanded parent). 4 new df R tests (all collapsed,
+      expanded/collapsed mix, backward compat, multi-level). 3 new DuckDB R tests (all
+      collapsed, expanded/collapsed mix, backward compat). All 515 JS tests pass, all R tests pass.
 
 #### 9F: Server-side data documentation