docs, xtask, common, exec: Get started holds the quickstart; every generated reference page has a description (website launch)

changelog.d/754.added.md

@@ -0,0 +1 @@
+The Get started page of the documentation now holds the quickstart, and every generated reference page (SQL functions, Python, Rust, configuration, metrics, Spark Connect coverage) carries a description.

crates/krishiv-cli/tests/check_compat.rs

@@ -451,7 +451,7 @@ fn an_unwritable_report_exits_2() {
 fn the_docs_examples_pass_check_compat() {
     let docs = repo_root().join("docs");
     let scratch = tempfile::tempdir().expect("tempdir");
-    for (i, page) in ["get-started/quickstart.md", "guides/pyspark-migration.md"]
+    for (i, page) in ["get-started/index.md", "guides/pyspark-migration.md"]
         .iter()
         .enumerate()
     {

crates/krishiv-common/src/config/mod.rs

@@ -647,7 +647,8 @@ pub fn docs_markdown() -> String {
     let mut entries: Vec<&ConfigEntry> = REGISTRY.to_vec();
     entries.sort_by_key(|e| e.key);
     let mut out = String::from(
-        "# Krishiv configuration reference\n\n\
+        "---\ndescription: \"Every krishiv.* configuration key, with its type, default, scope and Spark alias.\"\n---\n\n\
+         # Krishiv configuration reference\n\n\
          <!-- @generated by `cargo xtask docs-config`; do not edit. -->\n\n\
          Keys under `krishiv.experimental.` are experimental and off by default. \
          Spark aliases are accepted in place of the Krishiv key; other unknown \

crates/krishiv-common/src/config/tests.rs

@@ -613,7 +613,7 @@ fn registry_is_well_formed() {
 #[test]
 fn docs_markdown_lists_every_entry_deterministically() {
     let md = docs_markdown();
-    assert!(md.starts_with("# Krishiv configuration reference\n"));
+    assert!(md.contains("\n# Krishiv configuration reference\n"));
     assert!(md.contains(
         "| Key | Type | Default | Scope | Settable by | Since | Spark alias | Description |"
     ));

crates/krishiv-exec/src/metrics/registry.rs

@@ -897,7 +897,8 @@ impl MetricsRegistry {
     #[must_use]
     pub fn catalogue_markdown() -> String {
         let mut out = String::from(
-            "# Krishiv engine metrics (v1)\n\n\
+            "---\ndescription: \"Every engine metric: name, type, unit, labels and what it counts.\"\n---\n\n\
+             # Krishiv engine metrics (v1)\n\n\
              <!-- @generated by `cargo xtask docs-metrics`; do not edit. -->\n\n\
              Engine metrics, scraped per engine (`Session::metrics`, `krishiv.metrics()`): \
              OTel-style dotted names with the mechanical Prometheus rendering next to them \

docs/get-started/_meta.json

@@ -1,4 +1,3 @@
 {
-  "index": "Get started",
-  "quickstart": "Quickstart"
+  "index": "Get started"
 }

docs/get-started/index.md

@@ -1,14 +1,77 @@
 ---
-description: "Install Krishiv, run your first query and decide where to go next."
+description: "Install Krishiv, run your first SQL and DataFrame query, and decide where to go next."
 ---
 
 # Get started
 
-This section is for someone who has not run Krishiv yet. The [Quickstart](quickstart.md) installs the Python package,
-starts a session, runs a SQL query and a DataFrame query, and reads the result back as Arrow data. It takes a few
-minutes and needs Python 3.11 or newer.
+This section is for someone who has not run Krishiv yet. The steps below install the Python package, start a session,
+run a SQL query and a DataFrame query, and read the result back as Arrow data. They take a few minutes.
 
-Once it runs, pick the next step by what you came to do:
+<!-- The code blocks equal the Quickstart section of the repository README (a test checks it); CI runs the README's. -->
+
+Install the package with pyarrow for the Arrow results:
+
+```bash
+pip install "krishiv[pyarrow]"
+```
+
+Krishiv needs Python 3.11 or newer (the floor of the Spark 4.3 Python client); the examples run on 3.11 to 3.14. The `pyarrow` extra is needed for `collect()` and `toArrow()`, which return Arrow data.
+
+SQL and DataFrames:
+
+```python
+import tempfile
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+import krishiv
+from krishiv.sql import functions as F
+
+spark = krishiv.session()  # a PySpark-style SparkSession, in process
+
+# SQL.
+spark.sql("SELECT 1 AS answer").show()
+
+# DataFrames over a Parquet file; the same file is also a temp view for SQL.
+path = str(Path(tempfile.mkdtemp()) / "orders.parquet")
+pq.write_table(
+    pa.table(
+        {
+            "region": ["east", "west", "east", "north", "west"],
+            "amount": [10.0, 20.0, 5.5, 7.25, 30.0],
+        }
+    ),
+    path,
+)
+orders = spark.read.parquet(path)
+orders.createOrReplaceTempView("orders")
+
+by_region = (
+    orders.filter(F.col("amount") > 6)
+    .groupBy("region")
+    .agg(F.sum("amount").alias("revenue"))
+    .orderBy("region")
+)
+by_region.show()
+print(by_region.toArrow())  # a pyarrow.Table; also pa.table(by_region), polars.DataFrame(by_region)
+
+same = spark.sql("SELECT region, sum(amount) AS revenue FROM orders WHERE amount > 6 GROUP BY region ORDER BY region")
+assert same.collect() == by_region.collect()
+```
+
+`EXPLAIN` and `EXPLAIN ANALYZE` (per-operator rows, time, peak memory and spill bytes):
+
+```python
+by_region.explain()  # the physical plan
+by_region.explain(analyze=True)  # runs the query and prints each operator's metrics
+
+plan = spark.sql("EXPLAIN ANALYZE SELECT region, sum(amount) FROM orders GROUP BY region").collect()[0][0]
+assert plan.startswith("== Physical Plan (analyzed) ==")
+```
+
+## Next
 
 - Moving an existing Spark job: [migrate a PySpark job](../guides/pyspark-migration.md), and scan your code first with
   [check-compat](../guides/check-compat.md).

docs/guides/index.md

@@ -4,7 +4,7 @@ description: "Task-by-task guides for migrating, checking, deploying and diagnos
 
 # Guides
 
-Each guide solves one task and is written for someone who already ran the [Quickstart](../get-started/quickstart.md).
+Each guide solves one task and is written for someone who already ran the [Quickstart](../get-started/index.md).
 Moving a Spark job? Start with [Migrate a PySpark job](pyspark-migration.md) and run
 [check-compat](check-compat.md) over your project first. Choosing where Krishiv runs: [Deployment](deploy.md)
 (embedded, single-node, distributed). To work with it interactively, see

docs/index.md

@@ -14,7 +14,7 @@ write.
 ## Install
 
 Nothing is on PyPI yet, so build the wheel from the repository (`maturin build` in `python/`) and install it. Once
-released, `pip install "krishiv[pyarrow]"` is all it takes; the [Quickstart](get-started/quickstart.md) has the steps.
+released, `pip install "krishiv[pyarrow]"` is all it takes; the [Quickstart](get-started/index.md) has the steps.
 
 ## Where to start
 

docs/reference/config/index.md

@@ -1,3 +1,7 @@
+---
+description: "Every krishiv.* configuration key, with its type, default, scope and Spark alias."
+---
+
 # Krishiv configuration reference
 
 <!-- @generated by `cargo xtask docs-config`; do not edit. -->

docs/reference/connect/_meta.json

@@ -0,0 +1,4 @@
+{
+  "index": "Spark Connect",
+  "coverage": "Coverage"
+}

docs/reference/connect/coverage.md

@@ -1,3 +1,7 @@
+---
+description: "Which Spark Connect relations, commands and expressions Krishiv supports, one row per protocol arm."
+---
+
 <!-- Generated by `cargo xtask connect-coverage`; do not edit. -->
 # Spark Connect coverage
 

docs/reference/connect/index.md

@@ -0,0 +1,10 @@
+---
+description: "How much of the Spark Connect protocol Krishiv serves, arm by arm, for the vendored protocol version."
+---
+
+# Spark Connect
+
+Spark Connect lets a Spark client talk to an engine over gRPC. Krishiv serves a subset of the protocol today: the
+[coverage page](coverage.md) lists every relation, command and expression the translator dispatches on and whether
+Krishiv handles it, partly handles it, does not handle it yet or leaves it out of scope. A request that uses an arm
+Krishiv does not handle fails with `CONNECT_INVALID_PLAN.INVALID_ONE_OF_FIELD_NOT_SUPPORTED`.

docs/reference/metrics-v1.md

@@ -1,3 +1,7 @@
+---
+description: "Every engine metric: name, type, unit, labels and what it counts."
+---
+
 # Krishiv engine metrics (v1)
 
 <!-- @generated by `cargo xtask docs-metrics`; do not edit. -->

docs/reference/python/index.md

@@ -1,3 +1,7 @@
+---
+description: "Every public module of the krishiv Python package: classes, functions and signatures."
+---
+
 # Python API reference
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.errors.md

@@ -1,3 +1,7 @@
+---
+description: "Exceptions, shaped like `pyspark.errors` (Spark 4.2); no dependency on `pyspark`."
+---
+
 # `krishiv.errors`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.md

@@ -1,3 +1,7 @@
+---
+description: "Krishiv: a Rust-native, Spark-compatible batch and streaming engine."
+---
+
 # `krishiv`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.column.md

@@ -1,3 +1,7 @@
+---
+description: "`Column`: a column expression, as `pyspark.sql.Column` (a thin wrapper on the native handle)."
+---
+
 # `krishiv.sql.column`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.conf.md

@@ -1,3 +1,7 @@
+---
+description: "`RuntimeConfig`: `spark.conf`."
+---
+
 # `krishiv.sql.conf`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.dataframe.md

@@ -1,3 +1,7 @@
+---
+description: "`DataFrame`: a lazily evaluated query, as `pyspark.sql.DataFrame`."
+---
+
 # `krishiv.sql.dataframe`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.functions.md

@@ -1,3 +1,7 @@
+---
+description: "`pyspark.sql.functions`: column builders. A function resolves by name in the engine's"
+---
+
 # `krishiv.sql.functions`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.group.md

@@ -1,3 +1,7 @@
+---
+description: "`GroupedData`: a DataFrame grouped by key columns, as `pyspark.sql.GroupedData`."
+---
+
 # `krishiv.sql.group`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.md

@@ -1,3 +1,7 @@
+---
+description: "The PySpark-shaped API: `from krishiv.sql import functions as F`."
+---
+
 # `krishiv.sql`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.na.md

@@ -1,3 +1,7 @@
+---
+description: "`DataFrameNaFunctions` (`DataFrame.na`): handling of missing data, as `pyspark.sql.DataFrameNaFunctions`."
+---
+
 # `krishiv.sql.na`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.pandas.functions.md

@@ -1,3 +1,7 @@
+---
+description: "`pyspark.sql.pandas.functions`: `pandas_udf` and `arrow_udf`."
+---
+
 # `krishiv.sql.pandas.functions`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.pandas.md

@@ -1,3 +1,7 @@
+---
+description: "`pyspark.sql.pandas`: the pandas and Arrow vectorized UDFs (`krishiv.sql.pandas.functions`)."
+---
+
 # `krishiv.sql.pandas`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.profile_view.md

@@ -1,3 +1,7 @@
+---
+description: "An offline viewer for `krishiv.query-profile` documents."
+---
+
 # `krishiv.sql.profile_view`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.readwriter.md

@@ -1,3 +1,7 @@
+---
+description: "`DataFrameReader` (`spark.read`), `DataFrameWriter` (`df.write`)"
+---
+
 # `krishiv.sql.readwriter`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.row.md

@@ -1,3 +1,7 @@
+---
+description: "`Row`: a result row, as `pyspark.sql.Row` (a tuple with field names)."
+---
+
 # `krishiv.sql.row`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.session.md

@@ -1,3 +1,7 @@
+---
+description: "`SparkSession`: the entry point, as `pyspark.sql.SparkSession` (embedded: no server)."
+---
+
 # `krishiv.sql.session`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->

docs/reference/python/krishiv.sql.stat.md

@@ -1,3 +1,7 @@
+---
+description: "`DataFrameStatFunctions` (`DataFrame.stat`), as `pyspark.sql.DataFrameStatFunctions`."
+---
+
 # `krishiv.sql.stat`
 
 <!-- @generated by `python3 tests/scripts/gen_python_docs.py`; do not edit. -->