From c96d3a8a26b40018cb345f5e8d3f42069c1a3b38 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:30:47 +0100 Subject: [PATCH 01/21] ci, docs: drop the one-off nightly cron, RELEASING.md tag-fix section, 429 retry for the crate upload step Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .github/workflows/nightly.yml | 2 -- .github/workflows/publish-crates.yml | 16 ++++++++++++++-- RELEASING.md | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b690a10a9..2d4fc8523 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,8 +9,6 @@ on: schedule: # Off the hour: the top of the hour is when scheduled runs are delayed or dropped most. - cron: "23 22 * * *" - # One-off extra runs on 2026-11-13 for the v0.2.0 release (binaries.yml and release.yml still need a green scheduled nightly; publishing no longer does); remove after. - - cron: "17 8,10 13 11 *" workflow_dispatch: permissions: diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 1dc354cb2..658cf305a 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -40,7 +40,7 @@ jobs: needs: ci if: github.repository == 'KrishivAI/krishiv-engine' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 330 environment: name: crates-io url: https://crates.io/crates/krishiv @@ -72,6 +72,18 @@ jobs: continue fi echo "::group::$crate $version" - cargo publish -p "$crate" --locked # zizmor: ignore[use-trusted-publishing] scoped token in the gated crates-io environment + # crates.io limits how fast new crate names are created (429 "try again after "): wait and retry. + for attempt in $(seq 1 40); do + log="$RUNNER_TEMP/publish-$crate.log" + if cargo publish -p "$crate" --locked >"$log" 2>&1; then cat "$log"; break; fi # zizmor: ignore[use-trusted-publishing] scoped token in the gated crates-io environment + cat "$log" + grep -q '429 Too Many Requests' "$log" || exit 1 + [ "$attempt" -lt 40 ] || exit 1 + after=$(grep -oE 'try again after [^.]+GMT' "$log" | head -1 | sed 's/try again after //') + wait=$(( $( date -d "$after" +%s 2>/dev/null || echo $(( $(date +%s) + 600 )) ) - $(date +%s) + 20 )) + [ "$wait" -gt 20 ] || wait=60 + echo "rate limited on $crate (attempt $attempt): waiting ${wait}s until $after" + sleep "$wait" + done echo "::endgroup::" done < .github/scripts/publish-crates.txt diff --git a/RELEASING.md b/RELEASING.md index f5e5a1387..72e5b5f9d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -133,6 +133,25 @@ Each step is a hard stop for agents; the maintainer runs it and approves its env 5. **Yank the old 0.1 releases** once 0.2.0 is verified: PyPI `krishiv` 0.1.0; crates.io `krishiv` 0.1.0-nightly.202607030148 and `krishiv-common` 0.1.0 (`cargo yank --version `). See below. 6. **GitHub Release** for `v0.2.0`, with the CHANGELOG 0.2.0 section as its notes. +## Publish tag (a `.github`-only fix after tagging) + +A release commit sometimes needs a fix under `.github/` only (a workflow, a script) after it is tagged. **Never move +or delete the release tag** (`v0.2.0`): a published tag is permanent. + +1. Make one commit that changes only files under `.github/`, directly on the release commit (not on `main`). +2. Tag it `vX.Y.Z-publishN` (for 0.2.0: `v0.2.0-publish`, `-publish2`, `-publish3`) and push the tag. +3. Dispatch the publish workflows (`publish-pypi.yml`, `publish-crates.yml`, and `publish-testpypi.yml` for a + rehearsal) on that tag. `.github/scripts/require-green-nightly.sh` accepts exactly one `.github`-only commit on a + parent that a green scheduled nightly covers; any second commit, or a commit that touches other paths, is refused. + +Two things learned from 0.2.0: + +- **crates.io rate-limits new crate names** (HTTP 429, "try again after "). `publish-crates.yml` waits until + the stated time and retries each crate (up to 40 attempts, a 330-minute job limit); a crate already published is + skipped, so a failed run can also be dispatched again. +- **The bare `pip install krishiv` has no pyarrow**, and `collect()` needs it: install `krishiv[pyarrow]` + (the TestPyPI line above already does). + ## Yanking a bad release (owner only) Every step is a hard stop for agents; the maintainer runs them. Yanking hides a version from new resolves and From 0e12de012b04a20337d663349bea46532f660e68 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:59:34 +0100 Subject: [PATCH 02/21] python, krishiv: PyPI and crates.io metadata for search: readme, keywords, classifiers, project URLs Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv/Cargo.toml | 6 +++--- python/pyproject.toml | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/krishiv/Cargo.toml b/crates/krishiv/Cargo.toml index df6ccc481..10d8848a2 100644 --- a/crates/krishiv/Cargo.toml +++ b/crates/krishiv/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "krishiv" -description = "Spark-compatible SQL and DataFrame engine in Rust: embedded, vectorized on Arrow, no JVM" -keywords = ["spark", "sql", "dataframe", "arrow", "query-engine"] -categories = ["database-implementations", "data-structures"] +description = "Spark alternative in Rust: run Spark SQL and DataFrames embedded, vectorized on Arrow, no JVM" +keywords = ["spark", "pyspark", "sql", "dataframe", "query-engine"] +categories = ["database-implementations", "database"] homepage = "https://krishiv.ai" documentation = "https://docs.rs/krishiv" version.workspace = true diff --git a/python/pyproject.toml b/python/pyproject.toml index a9baf99c6..193b11193 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,10 +5,16 @@ build-backend = "maturin" [project] name = "krishiv" description = "Krishiv: a Rust-native, Spark-compatible batch and streaming engine" +readme = "README.md" requires-python = ">=3.11" license = { text = "Apache-2.0" } +keywords = ["spark", "pyspark", "spark-alternative", "sql", "dataframe", "arrow", "rust", "etl", "big-data"] dynamic = ["version"] classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Topic :: Database", + "Topic :: Scientific/Engineering", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.11", @@ -20,6 +26,13 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", ] +[project.urls] +Homepage = "https://krishiv.ai" +Documentation = "https://krishiv.ai/docs" +Benchmarks = "https://krishiv.ai/benchmarks" +Changelog = "https://github.com/KrishivAI/krishiv-engine/releases" +Source = "https://github.com/KrishivAI/krishiv-engine" + [project.optional-dependencies] # Arrow libraries are optional: `__arrow_c_stream__` needs none (Polars and DuckDB consume it # directly); pyarrow is needed by `toArrow`, which imports it lazily. `toPandas` needs the From 2069fc1459ada0b4bbea33738b21741f65af2c1c Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:14:58 +0100 Subject: [PATCH 03/21] release: 0.2.1 (PyPI.md, crate READMEs, pyarrow required, version bump, changelog) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- CHANGELOG.md | 14 ++++ Cargo.lock | 74 +++++++++++----------- Cargo.toml | 4 +- RELEASING.md | 4 +- crates/krishiv-alloc/Cargo.toml | 5 +- crates/krishiv-alloc/README.md | 7 ++ crates/krishiv-cli/Cargo.toml | 23 ++++--- crates/krishiv-cli/README.md | 7 ++ crates/krishiv-common/Cargo.toml | 3 + crates/krishiv-common/README.md | 7 ++ crates/krishiv-connect-server/Cargo.toml | 15 +++-- crates/krishiv-connect-server/README.md | 7 ++ crates/krishiv-connect/Cargo.toml | 9 ++- crates/krishiv-connect/README.md | 7 ++ crates/krishiv-connectors-delta/Cargo.toml | 6 +- crates/krishiv-exec/Cargo.toml | 21 +++--- crates/krishiv-exec/README.md | 7 ++ crates/krishiv-functions/Cargo.toml | 11 ++-- crates/krishiv-functions/README.md | 7 ++ crates/krishiv-io/Cargo.toml | 13 ++-- crates/krishiv-io/README.md | 7 ++ crates/krishiv-kernels/Cargo.toml | 7 +- crates/krishiv-kernels/README.md | 7 ++ crates/krishiv-mcp/Cargo.toml | 11 ++-- crates/krishiv-mcp/README.md | 7 ++ crates/krishiv-memory/Cargo.toml | 7 +- crates/krishiv-memory/README.md | 7 ++ crates/krishiv-optimizer/Cargo.toml | 15 +++-- crates/krishiv-optimizer/README.md | 7 ++ crates/krishiv-plan-codec/Cargo.toml | 13 ++-- crates/krishiv-plan-codec/README.md | 7 ++ crates/krishiv-plan/Cargo.toml | 15 +++-- crates/krishiv-plan/README.md | 7 ++ crates/krishiv-server/Cargo.toml | 11 ++-- crates/krishiv-server/README.md | 7 ++ crates/krishiv-session/Cargo.toml | 27 ++++---- crates/krishiv-session/README.md | 7 ++ crates/krishiv-sql/Cargo.toml | 11 ++-- crates/krishiv-sql/README.md | 7 ++ crates/krishiv-store/Cargo.toml | 5 +- crates/krishiv-store/README.md | 7 ++ crates/krishiv-types/Cargo.toml | 5 +- crates/krishiv-types/README.md | 7 ++ crates/krishiv-udf-sdk/Cargo.toml | 5 +- crates/krishiv-udf-sdk/README.md | 7 ++ crates/krishiv-ui-assets/Cargo.toml | 3 + crates/krishiv-ui-assets/README.md | 7 ++ crates/krishiv/Cargo.toml | 11 ++-- crates/krishiv/README.md | 52 +++++++++++++++ crates/krishiv/src/lib.rs | 5 ++ docs/reference/python-package.md | 2 +- python/PYPI.md | 65 +++++++++++++++++++ python/README.md | 3 +- python/pyproject.toml | 12 ++-- python/uv.lock | 8 +-- 55 files changed, 495 insertions(+), 147 deletions(-) create mode 100644 crates/krishiv-alloc/README.md create mode 100644 crates/krishiv-cli/README.md create mode 100644 crates/krishiv-common/README.md create mode 100644 crates/krishiv-connect-server/README.md create mode 100644 crates/krishiv-connect/README.md create mode 100644 crates/krishiv-exec/README.md create mode 100644 crates/krishiv-functions/README.md create mode 100644 crates/krishiv-io/README.md create mode 100644 crates/krishiv-kernels/README.md create mode 100644 crates/krishiv-mcp/README.md create mode 100644 crates/krishiv-memory/README.md create mode 100644 crates/krishiv-optimizer/README.md create mode 100644 crates/krishiv-plan-codec/README.md create mode 100644 crates/krishiv-plan/README.md create mode 100644 crates/krishiv-server/README.md create mode 100644 crates/krishiv-session/README.md create mode 100644 crates/krishiv-sql/README.md create mode 100644 crates/krishiv-store/README.md create mode 100644 crates/krishiv-types/README.md create mode 100644 crates/krishiv-udf-sdk/README.md create mode 100644 crates/krishiv-ui-assets/README.md create mode 100644 crates/krishiv/README.md create mode 100644 python/PYPI.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a4ce75ee..21f95721e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,20 @@ Later entries are generated by release-plz from Conventional Commits (`release-p Nothing yet. +## 0.2.1 + +A patch release: packaging and documentation only, no engine changes. + +### Changed + +- **`pyarrow>=16` is now a required dependency** of the Python package, so a bare `pip install krishiv` can `collect()` and + `show()`. `krishiv[pyarrow]` still works as an alias. +- The PyPI page (`krishiv` and `krishiv-compat`) now has a description (`python/PYPI.md`), keywords, classifiers and project URLs. +- The crates.io pages have READMEs: a full one for `krishiv` (its example runs as a doc test) and a short pointer to `krishiv` + for every internal crate; keywords and categories include `spark`. +- The crates.io publish workflow waits out the registry's 429 rate limit for new crate names and retries; the one-off + nightly cron is gone; `RELEASING.md` documents publishing from a tag. + ## 0.2.0 The first release is the embedded core: the engine runs in your process, from Python or Rust. diff --git a/Cargo.lock b/Cargo.lock index 561094a43..bd3db5ac7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2179,7 +2179,7 @@ dependencies = [ [[package]] name = "krishiv" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2193,7 +2193,7 @@ dependencies = [ [[package]] name = "krishiv-alloc" -version = "0.2.0" +version = "0.2.1" dependencies = [ "krishiv-common", "libmimalloc-sys", @@ -2201,7 +2201,7 @@ dependencies = [ [[package]] name = "krishiv-bench-runner" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-row", @@ -2223,7 +2223,7 @@ dependencies = [ [[package]] name = "krishiv-cli" -version = "0.2.0" +version = "0.2.1" dependencies = [ "axum", "clap", @@ -2248,7 +2248,7 @@ dependencies = [ [[package]] name = "krishiv-common" -version = "0.2.0" +version = "0.2.1" dependencies = [ "codspeed-criterion-compat", "serde_json", @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "krishiv-connect" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2274,7 +2274,7 @@ dependencies = [ [[package]] name = "krishiv-connect-server" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-ipc", @@ -2303,15 +2303,15 @@ dependencies = [ [[package]] name = "krishiv-connector-sdk" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-connectors-cdc" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-connectors-delta" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2335,15 +2335,15 @@ dependencies = [ [[package]] name = "krishiv-connectors-iceberg" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-connectors-kafka" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-exec" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2382,11 +2382,11 @@ dependencies = [ [[package]] name = "krishiv-flight-sql" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-functions" -version = "0.2.0" +version = "0.2.1" dependencies = [ "aes", "aes-gcm", @@ -2408,7 +2408,7 @@ dependencies = [ [[package]] name = "krishiv-io" -version = "0.2.0" +version = "0.2.1" dependencies = [ "apache-avro", "arrow-array", @@ -2445,7 +2445,7 @@ dependencies = [ [[package]] name = "krishiv-kernels" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2469,7 +2469,7 @@ dependencies = [ [[package]] name = "krishiv-mcp" -version = "0.2.0" +version = "0.2.1" dependencies = [ "futures", "krishiv-common", @@ -2484,7 +2484,7 @@ dependencies = [ [[package]] name = "krishiv-memory" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2505,7 +2505,7 @@ dependencies = [ [[package]] name = "krishiv-optimizer" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2524,7 +2524,7 @@ dependencies = [ [[package]] name = "krishiv-plan" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "krishiv-plan-codec" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-ipc", "arrow-schema", @@ -2564,7 +2564,7 @@ dependencies = [ [[package]] name = "krishiv-python" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2583,7 +2583,7 @@ dependencies = [ [[package]] name = "krishiv-scheduler" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2605,7 +2605,7 @@ dependencies = [ [[package]] name = "krishiv-server" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2623,7 +2623,7 @@ dependencies = [ [[package]] name = "krishiv-session" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2667,7 +2667,7 @@ dependencies = [ [[package]] name = "krishiv-shuffle" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-ipc", @@ -2682,7 +2682,7 @@ dependencies = [ [[package]] name = "krishiv-sim" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2706,7 +2706,7 @@ dependencies = [ [[package]] name = "krishiv-sql" -version = "0.2.0" +version = "0.2.1" dependencies = [ "codspeed-criterion-compat", "insta", @@ -2720,11 +2720,11 @@ dependencies = [ [[package]] name = "krishiv-state" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-store" -version = "0.2.0" +version = "0.2.1" dependencies = [ "async-trait", "bytes", @@ -2737,11 +2737,11 @@ dependencies = [ [[package]] name = "krishiv-stream" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-testkit" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2775,7 +2775,7 @@ dependencies = [ [[package]] name = "krishiv-types" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2789,7 +2789,7 @@ dependencies = [ [[package]] name = "krishiv-udf-sdk" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "krishiv-types", @@ -2797,7 +2797,7 @@ dependencies = [ [[package]] name = "krishiv-ui-assets" -version = "0.2.0" +version = "0.2.1" dependencies = [ "rust-embed", ] @@ -5420,7 +5420,7 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "xtask" -version = "0.2.0" +version = "0.2.1" dependencies = [ "krishiv-common", "krishiv-exec", diff --git a/Cargo.toml b/Cargo.toml index db86725cd..9ba528195 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,12 +3,14 @@ resolver = "3" members = ["crates/*", "xtask"] [workspace.package] -version = "0.2.0" +version = "0.2.1" edition = "2024" # MSRV = latest stable (1.99) minus two minor releases. rust-version = "1.97" license = "Apache-2.0" # ADR 0033 repository = "https://github.com/KrishivAI/krishiv-engine" +keywords = ["spark", "pyspark", "sql", "dataframe", "query-engine"] +categories = ["database-implementations", "database"] [workspace.lints.rust] unsafe_op_in_unsafe_fn = "deny" diff --git a/RELEASING.md b/RELEASING.md index 72e5b5f9d..b1cce73ea 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -149,8 +149,8 @@ Two things learned from 0.2.0: - **crates.io rate-limits new crate names** (HTTP 429, "try again after "). `publish-crates.yml` waits until the stated time and retries each crate (up to 40 attempts, a 330-minute job limit); a crate already published is skipped, so a failed run can also be dispatched again. -- **The bare `pip install krishiv` has no pyarrow**, and `collect()` needs it: install `krishiv[pyarrow]` - (the TestPyPI line above already does). +- **0.2.0's bare `pip install krishiv` had no pyarrow**, and `collect()` needs it. From 0.2.1 pyarrow is a + required dependency; `krishiv[pyarrow]` stays as an alias. ## Yanking a bad release (owner only) diff --git a/crates/krishiv-alloc/Cargo.toml b/crates/krishiv-alloc/Cargo.toml index cb2d487cb..923cbcc49 100644 --- a/crates/krishiv-alloc/Cargo.toml +++ b/crates/krishiv-alloc/Cargo.toml @@ -1,18 +1,21 @@ [package] name = "krishiv-alloc" description = "mimalloc settings and memory release for Krishiv binaries (never linked by libraries)" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] workspace = true [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } libmimalloc-sys = { workspace = true, features = ["extended"] } [features] diff --git a/crates/krishiv-alloc/README.md b/crates/krishiv-alloc/README.md new file mode 100644 index 000000000..d6073de2f --- /dev/null +++ b/crates/krishiv-alloc/README.md @@ -0,0 +1,7 @@ +# krishiv-alloc + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-cli/Cargo.toml b/crates/krishiv-cli/Cargo.toml index 1e5001acb..8639b120d 100644 --- a/crates/krishiv-cli/Cargo.toml +++ b/crates/krishiv-cli/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-cli" description = "Krishiv command line binary" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [[bin]] @@ -18,17 +21,17 @@ doc = false [dependencies] clap = { workspace = true, features = ["std", "derive", "help", "usage", "error-context"] } futures = { workspace = true, features = ["alloc", "executor"] } -krishiv = { path = "../krishiv", version = "=0.2.0" } -krishiv-alloc = { path = "../krishiv-alloc", version = "=0.2.0", optional = true } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0", optional = true } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-sql = { path = "../krishiv-sql", version = "=0.2.0" } +krishiv = { path = "../krishiv", version = "=0.2.1" } +krishiv-alloc = { path = "../krishiv-alloc", version = "=0.2.1", optional = true } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1", optional = true } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-sql = { path = "../krishiv-sql", version = "=0.2.1" } reedline = { workspace = true } -krishiv-server = { path = "../krishiv-server", version = "=0.2.0", optional = true } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0", optional = true } -krishiv-mcp = { path = "../krishiv-mcp", version = "=0.2.0", optional = true } -krishiv-connect-server = { path = "../krishiv-connect-server", version = "=0.2.0", optional = true } +krishiv-server = { path = "../krishiv-server", version = "=0.2.1", optional = true } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1", optional = true } +krishiv-mcp = { path = "../krishiv-mcp", version = "=0.2.1", optional = true } +krishiv-connect-server = { path = "../krishiv-connect-server", version = "=0.2.1", optional = true } axum = { workspace = true, optional = true } tokio = { workspace = true, features = ["rt", "rt-multi-thread"], optional = true } serde_json = { workspace = true } diff --git a/crates/krishiv-cli/README.md b/crates/krishiv-cli/README.md new file mode 100644 index 000000000..8eb5bb8c8 --- /dev/null +++ b/crates/krishiv-cli/README.md @@ -0,0 +1,7 @@ +# krishiv-cli + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-common/Cargo.toml b/crates/krishiv-common/Cargo.toml index 0ac119ea1..a59332ab7 100644 --- a/crates/krishiv-common/Cargo.toml +++ b/crates/krishiv-common/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-common" description = "Errors, config registry and tracing setup" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] diff --git a/crates/krishiv-common/README.md b/crates/krishiv-common/README.md new file mode 100644 index 000000000..0894408cb --- /dev/null +++ b/crates/krishiv-common/README.md @@ -0,0 +1,7 @@ +# krishiv-common + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-connect-server/Cargo.toml b/crates/krishiv-connect-server/Cargo.toml index 74a71574c..b39717733 100644 --- a/crates/krishiv-connect-server/Cargo.toml +++ b/crates/krishiv-connect-server/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-connect-server" description = "Spark Connect gRPC server: the service, session manager and limits" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -17,12 +20,12 @@ arrow-ipc = { workspace = true, features = ["lz4", "zstd"] } arrow-schema.workspace = true futures = { workspace = true, features = ["std"] } bytes = { workspace = true, features = ["std"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-connect = { path = "../krishiv-connect", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0", features = ["unstable-ir"] } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-connect = { path = "../krishiv-connect", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1", features = ["unstable-ir"] } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } getrandom.workspace = true prost = { workspace = true, features = ["derive", "std"] } prost-types = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-connect-server/README.md b/crates/krishiv-connect-server/README.md new file mode 100644 index 000000000..42a7d2400 --- /dev/null +++ b/crates/krishiv-connect-server/README.md @@ -0,0 +1,7 @@ +# krishiv-connect-server + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-connect/Cargo.toml b/crates/krishiv-connect/Cargo.toml index 003fa86ed..2647bba73 100644 --- a/crates/krishiv-connect/Cargo.toml +++ b/crates/krishiv-connect/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-connect" description = "Spark Connect server" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -14,9 +17,9 @@ workspace = true [dependencies] arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } # `no-recursion-limit`: prost's fixed limit of 100 nested messages rejects # ordinary plans (a loop of `withColumn` calls); `wire::decode_plan` bounds the # depth itself, before prost recurses (see the crate CLAUDE.md). diff --git a/crates/krishiv-connect/README.md b/crates/krishiv-connect/README.md new file mode 100644 index 000000000..2ac993cfd --- /dev/null +++ b/crates/krishiv-connect/README.md @@ -0,0 +1,7 @@ +# krishiv-connect + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-connectors-delta/Cargo.toml b/crates/krishiv-connectors-delta/Cargo.toml index 7c04e7798..0bfcaed8e 100644 --- a/crates/krishiv-connectors-delta/Cargo.toml +++ b/crates/krishiv-connectors-delta/Cargo.toml @@ -22,9 +22,9 @@ arrow-select.workspace = true # `internal-api` has no dependencies. delta_kernel = { workspace = true, features = ["internal-api"] } bytes.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } object_store.workspace = true serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["macros", "rt", "sync"] } diff --git a/crates/krishiv-exec/Cargo.toml b/crates/krishiv-exec/Cargo.toml index 04f374814..bbeccef4e 100644 --- a/crates/krishiv-exec/Cargo.toml +++ b/crates/krishiv-exec/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-exec" description = "Batch execution engine" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -21,15 +24,15 @@ arrow-select.workspace = true foldhash.workspace = true hashbrown.workspace = true futures = { workspace = true, features = ["alloc"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } -krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } +krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } thiserror.workspace = true # The query profile's JSON document (`profile`). serde_json = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-exec/README.md b/crates/krishiv-exec/README.md new file mode 100644 index 000000000..35189ba58 --- /dev/null +++ b/crates/krishiv-exec/README.md @@ -0,0 +1,7 @@ +# krishiv-exec + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-functions/Cargo.toml b/crates/krishiv-functions/Cargo.toml index 6f48a1cdb..6eca37ada 100644 --- a/crates/krishiv-functions/Cargo.toml +++ b/crates/krishiv-functions/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-functions" description = "Spark-compatible function library" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] @@ -26,10 +29,10 @@ regex-automata = { workspace = true, features = [ "unicode-word-boundary", ] } regex-syntax = { workspace = true, features = ["unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } quick-xml.workspace = true aes.workspace = true cbc = { workspace = true, features = ["alloc", "block-padding"] } diff --git a/crates/krishiv-functions/README.md b/crates/krishiv-functions/README.md new file mode 100644 index 000000000..53630a09a --- /dev/null +++ b/crates/krishiv-functions/README.md @@ -0,0 +1,7 @@ +# krishiv-functions + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-io/Cargo.toml b/crates/krishiv-io/Cargo.toml index c68ef1307..d1342b405 100644 --- a/crates/krishiv-io/Cargo.toml +++ b/crates/krishiv-io/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-io" description = "Object store and file format I/O" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -30,13 +33,13 @@ serde_json = { workspace = true, features = ["std"] } # Exact runtime-filter sets, placed by the row hash (P2-T163). hashbrown.workspace = true futures = { workspace = true, features = ["alloc", "std"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } # Spark value text conversions and datetime patterns (CSV, JSON, text). -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } # URL parsing, store building and pacing budgets (ADR 0044); `cloud` turns on its builders. -krishiv-store = { path = "../krishiv-store", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-store = { path = "../krishiv-store", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } memchr.workspace = true # `fs`: the local store reads use wraps `LocalFileSystem` (`io/local_listing.rs`). object_store = { workspace = true, features = ["fs"] } diff --git a/crates/krishiv-io/README.md b/crates/krishiv-io/README.md new file mode 100644 index 000000000..fe7ea4bdb --- /dev/null +++ b/crates/krishiv-io/README.md @@ -0,0 +1,7 @@ +# krishiv-io + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-kernels/Cargo.toml b/crates/krishiv-kernels/Cargo.toml index bd9995464..6998ab96e 100644 --- a/crates/krishiv-kernels/Cargo.toml +++ b/crates/krishiv-kernels/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-kernels" description = "Compute kernels (SIMD)" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -18,8 +21,8 @@ arrow-schema.workspace = true arrow-select.workspace = true arrow-string.workspace = true arrow-ord.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } jiff = { workspace = true, features = ["std", "tzdb-bundle-always"] } libm.workspace = true memchr = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-kernels/README.md b/crates/krishiv-kernels/README.md new file mode 100644 index 000000000..8c24a8b5f --- /dev/null +++ b/crates/krishiv-kernels/README.md @@ -0,0 +1,7 @@ +# krishiv-kernels + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-mcp/Cargo.toml b/crates/krishiv-mcp/Cargo.toml index 35c8df4a9..b5f13f26b 100644 --- a/crates/krishiv-mcp/Cargo.toml +++ b/crates/krishiv-mcp/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-mcp" description = "MCP adapter" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -14,10 +17,10 @@ workspace = true [dependencies] rmcp = { workspace = true, features = ["server", "transport-io"] } futures.workspace = true -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0" } -krishiv-sql = { path = "../krishiv-sql", version = "=0.2.0" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1" } +krishiv-sql = { path = "../krishiv-sql", version = "=0.2.1" } serde_json = { workspace = true, features = ["std"] } # The `mcp_stdio` conformance shim serves stdio from a plain binary (not a # test harness), so its stdout carries only JSON-RPC. diff --git a/crates/krishiv-mcp/README.md b/crates/krishiv-mcp/README.md new file mode 100644 index 000000000..95e009cad --- /dev/null +++ b/crates/krishiv-mcp/README.md @@ -0,0 +1,7 @@ +# krishiv-mcp + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-memory/Cargo.toml b/crates/krishiv-memory/Cargo.toml index b28280f4a..20b6ad07e 100644 --- a/crates/krishiv-memory/Cargo.toml +++ b/crates/krishiv-memory/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-memory" description = "Memory management" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -17,8 +20,8 @@ arrow-buffer.workspace = true arrow-ipc = { workspace = true, features = ["lz4", "zstd"] } arrow-schema.workspace = true sha2 = { workspace = true } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } thiserror.workspace = true tracing = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-memory/README.md b/crates/krishiv-memory/README.md new file mode 100644 index 000000000..982eb6816 --- /dev/null +++ b/crates/krishiv-memory/README.md @@ -0,0 +1,7 @@ +# krishiv-memory + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-optimizer/Cargo.toml b/crates/krishiv-optimizer/Cargo.toml index 05de1850d..31698faf8 100644 --- a/crates/krishiv-optimizer/Cargo.toml +++ b/crates/krishiv-optimizer/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "krishiv-optimizer" description = "Query optimizer" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } [dev-dependencies] criterion.workspace = true diff --git a/crates/krishiv-optimizer/README.md b/crates/krishiv-optimizer/README.md new file mode 100644 index 000000000..8fd3bfb0f --- /dev/null +++ b/crates/krishiv-optimizer/README.md @@ -0,0 +1,7 @@ +# krishiv-optimizer + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-plan-codec/Cargo.toml b/crates/krishiv-plan-codec/Cargo.toml index 29edefc78..617233674 100644 --- a/crates/krishiv-plan-codec/Cargo.toml +++ b/crates/krishiv-plan-codec/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-plan-codec" description = "The krishiv.plan.v1 codec for physical plans" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -14,11 +17,11 @@ workspace = true [dependencies] arrow-ipc.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } prost = { workspace = true, features = ["derive", "std"] } thiserror.workspace = true diff --git a/crates/krishiv-plan-codec/README.md b/crates/krishiv-plan-codec/README.md new file mode 100644 index 000000000..c0a31e577 --- /dev/null +++ b/crates/krishiv-plan-codec/README.md @@ -0,0 +1,7 @@ +# krishiv-plan-codec + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-plan/Cargo.toml b/crates/krishiv-plan/Cargo.toml index bd7068d9f..a8dbb7498 100644 --- a/crates/krishiv-plan/Cargo.toml +++ b/crates/krishiv-plan/Cargo.toml @@ -1,27 +1,30 @@ [package] name = "krishiv-plan" description = "Logical and physical plans" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] # One-row arrays: constant arguments fold through the cast kernels. arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } # The built-in function registry and the signature types it shares. -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } # The CSV literal inference `schema_of_csv` resolves with (layer L2). -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } # Spark's decimal result types, shared with the kernels that compute them. -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } # Session UDFs, for constant folding through the registry. -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } thiserror.workspace = true twox-hash = { workspace = true, features = ["xxhash3_128"] } diff --git a/crates/krishiv-plan/README.md b/crates/krishiv-plan/README.md new file mode 100644 index 000000000..cdba623cd --- /dev/null +++ b/crates/krishiv-plan/README.md @@ -0,0 +1,7 @@ +# krishiv-plan + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-server/Cargo.toml b/crates/krishiv-server/Cargo.toml index e139475e2..66a09cb6f 100644 --- a/crates/krishiv-server/Cargo.toml +++ b/crates/krishiv-server/Cargo.toml @@ -1,21 +1,24 @@ [package] name = "krishiv-server" description = "Admin and UI API server" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] axum = { workspace = true, features = ["http1", "tokio"] } arrow-array = { workspace = true, optional = true } arrow-schema = { workspace = true, optional = true } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0", optional = true } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0", optional = true } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0", optional = true } -krishiv-ui-assets = { path = "../krishiv-ui-assets", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1", optional = true } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1", optional = true } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1", optional = true } +krishiv-ui-assets = { path = "../krishiv-ui-assets", version = "=0.2.1" } futures = { workspace = true, features = ["alloc"] } getrandom.workspace = true serde_json = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-server/README.md b/crates/krishiv-server/README.md new file mode 100644 index 000000000..4b43b275e --- /dev/null +++ b/crates/krishiv-server/README.md @@ -0,0 +1,7 @@ +# krishiv-server + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-session/Cargo.toml b/crates/krishiv-session/Cargo.toml index 03dc0c028..78794d610 100644 --- a/crates/krishiv-session/Cargo.toml +++ b/crates/krishiv-session/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-session" description = "Krishiv sessions: SQL and DataFrames over the engine" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] @@ -15,18 +18,18 @@ arrow-schema.workspace = true thiserror = { workspace = true, optional = true } # `executor`: file footers are read on a thread of their own (catalog.rs). futures = { workspace = true, features = ["alloc", "executor"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } -krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-plan-codec = { path = "../krishiv-plan-codec", version = "=0.2.0" } -krishiv-sql = { path = "../krishiv-sql", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } +krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-plan-codec = { path = "../krishiv-plan-codec", version = "=0.2.1" } +krishiv-sql = { path = "../krishiv-sql", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } # The `EXPLAIN VERBOSE` JSON document. serde_json = { workspace = true, features = ["std"] } # Static token digests (`auth/tokens.rs`, P3-T002). diff --git a/crates/krishiv-session/README.md b/crates/krishiv-session/README.md new file mode 100644 index 000000000..2be836949 --- /dev/null +++ b/crates/krishiv-session/README.md @@ -0,0 +1,7 @@ +# krishiv-session + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-sql/Cargo.toml b/crates/krishiv-sql/Cargo.toml index 5893b7c29..b13edfe6d 100644 --- a/crates/krishiv-sql/Cargo.toml +++ b/crates/krishiv-sql/Cargo.toml @@ -1,21 +1,24 @@ [package] name = "krishiv-sql" description = "SQL parsing and analysis" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] workspace = true [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } thiserror.workspace = true [dev-dependencies] diff --git a/crates/krishiv-sql/README.md b/crates/krishiv-sql/README.md new file mode 100644 index 000000000..734bf3e7c --- /dev/null +++ b/crates/krishiv-sql/README.md @@ -0,0 +1,7 @@ +# krishiv-sql + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-store/Cargo.toml b/crates/krishiv-store/Cargo.toml index 837973789..8bca672e7 100644 --- a/crates/krishiv-store/Cargo.toml +++ b/crates/krishiv-store/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-store" description = "Object-store layer shared by the Krishiv engine and platform: URLs, store building, credentials (ADR 0044)" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true # Meant to be published (ADR 0044); stays unpublished until the owner's first release of it. publish = true @@ -14,7 +17,7 @@ workspace = true # krishiv-common and object_store only (layer L1, ADR 0044). [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } # Bearer header of `hf://` reads; only with `cloud`. http = { workspace = true, optional = true } object_store.workspace = true diff --git a/crates/krishiv-store/README.md b/crates/krishiv-store/README.md new file mode 100644 index 000000000..a20538941 --- /dev/null +++ b/crates/krishiv-store/README.md @@ -0,0 +1,7 @@ +# krishiv-store + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-types/Cargo.toml b/crates/krishiv-types/Cargo.toml index f8a9e9e8f..863c5c25e 100644 --- a/crates/krishiv-types/Cargo.toml +++ b/crates/krishiv-types/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-types" description = "Internal type representations" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -16,7 +19,7 @@ arrow-array.workspace = true arrow-buffer.workspace = true arrow-schema.workspace = true arrow-select.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } serde_json = { workspace = true, features = ["std"] } [dev-dependencies] diff --git a/crates/krishiv-types/README.md b/crates/krishiv-types/README.md new file mode 100644 index 000000000..557f1c397 --- /dev/null +++ b/crates/krishiv-types/README.md @@ -0,0 +1,7 @@ +# krishiv-types + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-udf-sdk/Cargo.toml b/crates/krishiv-udf-sdk/Cargo.toml index 465593f08..06c1ff91d 100644 --- a/crates/krishiv-udf-sdk/Cargo.toml +++ b/crates/krishiv-udf-sdk/Cargo.toml @@ -1,16 +1,19 @@ [package] name = "krishiv-udf-sdk" description = "UDF SDK" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] arrow-array.workspace = true -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } [lints] workspace = true diff --git a/crates/krishiv-udf-sdk/README.md b/crates/krishiv-udf-sdk/README.md new file mode 100644 index 000000000..c4d26b113 --- /dev/null +++ b/crates/krishiv-udf-sdk/README.md @@ -0,0 +1,7 @@ +# krishiv-udf-sdk + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-ui-assets/Cargo.toml b/crates/krishiv-ui-assets/Cargo.toml index 4265befb0..e0e352ffc 100644 --- a/crates/krishiv-ui-assets/Cargo.toml +++ b/crates/krishiv-ui-assets/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-ui-assets" description = "Embedded web UI assets" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] diff --git a/crates/krishiv-ui-assets/README.md b/crates/krishiv-ui-assets/README.md new file mode 100644 index 000000000..ef7f8136e --- /dev/null +++ b/crates/krishiv-ui-assets/README.md @@ -0,0 +1,7 @@ +# krishiv-ui-assets + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv/Cargo.toml b/crates/krishiv/Cargo.toml index 10d8848a2..903beef56 100644 --- a/crates/krishiv/Cargo.toml +++ b/crates/krishiv/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "krishiv" description = "Spark alternative in Rust: run Spark SQL and DataFrames embedded, vectorized on Arrow, no JVM" +readme = "README.md" keywords = ["spark", "pyspark", "sql", "dataframe", "query-engine"] categories = ["database-implementations", "database"] homepage = "https://krishiv.ai" @@ -18,11 +19,11 @@ workspace = true [dependencies] arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0", optional = true } -krishiv-server = { path = "../krishiv-server", version = "=0.2.0", optional = true } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1", optional = true } +krishiv-server = { path = "../krishiv-server", version = "=0.2.1", optional = true } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } [dev-dependencies] futures = { workspace = true, features = ["alloc", "executor"] } diff --git a/crates/krishiv/README.md b/crates/krishiv/README.md new file mode 100644 index 000000000..507a2a707 --- /dev/null +++ b/crates/krishiv/README.md @@ -0,0 +1,52 @@ +# krishiv + +**Spark alternative in Rust: run Spark SQL and DataFrames embedded, vectorized on Arrow, no JVM.** + +Krishiv runs inside your process: no cluster and no JVM. Queries follow Spark's SQL and DataFrame semantics and +error classes, execute vectorized on Apache Arrow, and return Arrow record batches. + +```bash +cargo add krishiv +``` + +```rust +use std::sync::Arc; +use krishiv::arrow::array::{Int32Array, RecordBatch}; +use krishiv::arrow::datatypes::{DataType, Field, Schema}; +use krishiv::functions::col; +use krishiv::Session; + +let session = Session::new()?; +let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); +let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![3, 1, 2]))]).unwrap(); +session.register_batches("t", schema, vec![batch])?; + +// DataFrames and SQL run on the same engine. +let top = session.table("t").order_by([col("a").desc()]).limit(2); +let rows: Vec = futures::executor::block_on(top.collect())?; +assert_eq!(rows.iter().map(|b| b.num_rows()).sum::(), 2); +# Ok::<(), krishiv::Error>(()) +``` + +Actions are `async` and run on the session's own executor, so any runtime (or none, with +`futures::executor::block_on`) can await them. + +## Speed + +On TPC-H at scale factor 100, Krishiv is **10.4x faster than Spark 4.2.0** (derived from TPC-H, not an official +TPC result). Method and numbers: . + +## Status + +Alpha. The embedded batch core works; streaming, incremental views, Spark Connect and distributed execution are +still being built. There is no stable API yet. Python users: `pip install krishiv`. + +## Links + +- Website: +- Documentation: +- Source and issues: +- Releases and changelog: + +Licensed under Apache-2.0. Apache Spark is a trademark of the Apache Software Foundation; Krishiv is not +affiliated with it. diff --git a/crates/krishiv/src/lib.rs b/crates/krishiv/src/lib.rs index 333f7802d..5812b5068 100644 --- a/crates/krishiv/src/lib.rs +++ b/crates/krishiv/src/lib.rs @@ -29,6 +29,11 @@ #![forbid(unsafe_code)] +// The crate README is a doc test: its example is compiled and run by `cargo test --doc`. +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeDoctests; + pub use krishiv_common::error::KrishivError as Error; pub use krishiv_session::{ AggregateProperties, AggregateUdf, ArgField, CancelReason, Column, CounterSample, DataFrame, diff --git a/docs/reference/python-package.md b/docs/reference/python-package.md index 9bfbe847c..797446837 100644 --- a/docs/reference/python-package.md +++ b/docs/reference/python-package.md @@ -16,7 +16,7 @@ table = df.toArrow() # a pyarrow.Table - Actions (`collect`, `show`, `toArrow`, `toPandas`, `count`) run the query in-process with the GIL released. - A DataFrame implements the Arrow PyCapsule interface, so `pa.table(df)`, `polars.DataFrame(df)` and `duckdb.from_arrow(df)` read the result without copying. -- pyarrow is optional (`pip install "krishiv[pyarrow]"`); `toPandas` also needs pandas (`krishiv[pandas]`). +- pyarrow is a required dependency (since 0.2.1; `krishiv[pyarrow]` still works as an alias); `toPandas` also needs pandas (`krishiv[pandas]`). - `df.profile()` (or `krishiv.ui(df)`) returns a viewer of the query's profile; in a notebook it renders itself. - Test your PySpark code without a JVM: `krishiv.testing` has `assertDataFrameEqual`, `assertSchemaEqual` and a pytest fixture (`pytest_plugins = ["krishiv.testing.pytest_plugin"]`, then use the `krishiv_session` fixture). diff --git a/python/PYPI.md b/python/PYPI.md new file mode 100644 index 000000000..0c0855afe --- /dev/null +++ b/python/PYPI.md @@ -0,0 +1,65 @@ +# Krishiv + +**Spark alternative in Rust: PySpark and Spark SQL without the JVM.** + +Krishiv runs Spark SQL and a PySpark-style DataFrame API inside your Python process. No cluster, no JVM, no +server to start. Execution is vectorized on Apache Arrow, and results come back as Arrow tables you can hand to +pandas, Polars or DuckDB without copying. + +## Install + +```bash +pip install krishiv +``` + +Needs Python 3.11 or newer. `pyarrow` is installed with it. For `toPandas()` add the extra: +`pip install "krishiv[pandas]"`. + +## 30-second example + +```python +import krishiv +from krishiv.sql import functions as F + +spark = krishiv.session() # a PySpark-style SparkSession, in process + +spark.sql("SELECT 1 AS answer").show() + +df = spark.sql("SELECT * FROM VALUES ('east', 10.0), ('west', 20.0), ('east', 5.5) AS t(region, amount)") +df.groupBy("region").agg(F.sum("amount").alias("revenue")).orderBy("region").show() + +table = df.toArrow() # a pyarrow.Table; also pa.table(df), polars.DataFrame(df), duckdb.from_arrow(df) +``` + +## What works in 0.2 + +- Batch Spark SQL and DataFrames, in process, over Arrow. `krishiv.sql` mirrors `pyspark.sql` + (import paths and signatures); errors in `krishiv.errors` carry Spark's error classes. +- Reading and writing Parquet files; `EXPLAIN` and `EXPLAIN ANALYZE` with per-operator rows, time and memory. +- Spark's function library, with Spark's results (hashing, casts, null handling and error behaviour follow Spark). +- `krishiv.testing`: `assertDataFrameEqual` and a pytest fixture, to run your PySpark tests without a JVM. + +Streaming, incremental views, Spark Connect and distributed execution are still being built. This is an alpha: +there is no stable API yet. + +## Speed + +On TPC-H at scale factor 100, Krishiv is **10.4x faster than Spark 4.2.0** (derived from TPC-H, not an official +TPC result). Method, queries and raw numbers: . + +## Supported CPUs + +The default wheels need an x86-64-v3 CPU (Intel Haswell, 2013, or AMD Zen and newer) or, on arm64, Armv8.2 (AWS +Graviton2, Apple silicon). On an older x86-64 CPU install `krishiv-compat` instead: the same package and the +same `import krishiv`, built for the plain x86-64 baseline (Linux only). Install one of the two, not both. + +## Links + +- Website: +- Documentation: +- Benchmarks: +- Source and issues: +- Releases and changelog: + +Krishiv is licensed under Apache-2.0. Apache Spark is a trademark of the Apache Software Foundation; Krishiv is +not affiliated with it. diff --git a/python/README.md b/python/README.md index 5afacb89e..16d0a65a1 100644 --- a/python/README.md +++ b/python/README.md @@ -29,8 +29,7 @@ DataFrame also implements the Arrow PyCapsule interface (`__arrow_c_stream__`), and releasing the stream cancels it. `krishiv.sql` mirrors `pyspark.sql` (signatures and import paths); errors are in `krishiv.errors`, shaped like -`pyspark.errors`. pyarrow is optional: install `krishiv[pyarrow]`; `toArrow`, `collect`, `show` and `toPandas` need it (`count` and -the PyCapsule interface do not). `toPandas` also needs pandas (`pip install krishiv[pandas]`). SQL expression strings +`pyspark.errors`. pyarrow is a required dependency since 0.2.1 (`krishiv[pyarrow]` still works as an alias); `toArrow`, `collect`, `show` and `toPandas` use it. `toPandas` also needs pandas (`pip install krishiv[pandas]`). SQL expression strings are supported (`F.expr`, `selectExpr`, `filter`/`where` with a string). ## Run your PySpark tests locally diff --git a/python/pyproject.toml b/python/pyproject.toml index 193b11193..ed22b1ee4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,11 +4,12 @@ build-backend = "maturin" [project] name = "krishiv" -description = "Krishiv: a Rust-native, Spark-compatible batch and streaming engine" -readme = "README.md" +description = "Spark alternative in Rust: PySpark and Spark SQL without the JVM" +readme = "PYPI.md" requires-python = ">=3.11" license = { text = "Apache-2.0" } keywords = ["spark", "pyspark", "spark-alternative", "sql", "dataframe", "arrow", "rust", "etl", "big-data"] +dependencies = ["pyarrow>=16"] dynamic = ["version"] classifiers = [ "Development Status :: 3 - Alpha", @@ -34,10 +35,9 @@ Changelog = "https://github.com/KrishivAI/krishiv-engine/releases" Source = "https://github.com/KrishivAI/krishiv-engine" [project.optional-dependencies] -# Arrow libraries are optional: `__arrow_c_stream__` needs none (Polars and DuckDB consume it -# directly); pyarrow is needed by `toArrow`, which imports it lazily. `toPandas` needs the -# `pandas` extra (pandas and numpy; not bundled in the wheel). -pyarrow = ["pyarrow>=16"] +# pyarrow is a required dependency since 0.2.1; this extra stays so `krishiv[pyarrow]` install lines still work. +# `toPandas` needs the `pandas` extra (pandas and numpy; not bundled in the wheel). +pyarrow = [] pandas = ["pandas>=2.2", "numpy>=1.26", "pyarrow>=16"] [dependency-groups] diff --git a/python/uv.lock b/python/uv.lock index 8b4ebaff0..a346267b7 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -189,6 +189,9 @@ wheels = [ [[package]] name = "krishiv" source = { editable = "." } +dependencies = [ + { name = "pyarrow" }, +] [package.optional-dependencies] pandas = [ @@ -197,9 +200,6 @@ pandas = [ { name = "pandas" }, { name = "pyarrow" }, ] -pyarrow = [ - { name = "pyarrow" }, -] [package.dev-dependencies] dev = [ @@ -216,8 +216,8 @@ dev = [ requires-dist = [ { name = "numpy", marker = "extra == 'pandas'", specifier = ">=1.26" }, { name = "pandas", marker = "extra == 'pandas'", specifier = ">=2.2" }, + { name = "pyarrow", specifier = ">=16" }, { name = "pyarrow", marker = "extra == 'pandas'", specifier = ">=16" }, - { name = "pyarrow", marker = "extra == 'pyarrow'", specifier = ">=16" }, ] provides-extras = ["pandas", "pyarrow"] From bdee35d91888ab74d88fd26b17f7141d5b39940c Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:15:46 +0100 Subject: [PATCH 04/21] README: the 0.3 roadmap row and planned example say single-node (krishiv server, Spark Connect) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dddeb9963..4c8452163 100644 --- a/README.md +++ b/README.md @@ -118,13 +118,13 @@ musllinux compat build is provided: ## Status and roadmap at a glance -Krishiv is at **`0.2.0`**. The embedded core (in-process batch SQL and DataFrames over Arrow, from Python) runs; +Krishiv is at **`0.2.1`**. The embedded core (in-process batch SQL and DataFrames over Arrow, from Python) runs; streaming, incremental views, Spark Connect and distributed execution are still being built. | Milestone | Focus | Status | |---|---|---| | 1. Embedded core alpha | In-process SQL and DataFrames on Arrow, with Python and Rust APIs | 🚧 In progress | -| 2. Spark Connect and lakehouse | PySpark over Spark Connect; Parquet, Iceberg and Delta | ⏳ Planned | +| 2. Single-node (0.3) | `krishiv server` with Spark Connect, authentication, Iceberg and Delta, Python UDFs | ⏳ Planned | | 3. Streaming | Stateful streaming with the same SQL and semantics as batch | ⏳ Planned | | 4. Incremental views | Always-fresh materialized views and queryable state | ⏳ Planned | | 5. Distributed | Scale out to a cluster with the same code | ⏳ Planned | @@ -216,7 +216,7 @@ Spark. ```python from pyspark.sql import SparkSession, functions as F -# Planned: a local Krishiv server speaking the Spark Connect protocol. +# Planned: single-node Krishiv (`krishiv server`) speaking the Spark Connect protocol. spark = SparkSession.builder.remote("sc://localhost:15002").getOrCreate() orders = spark.read.parquet("s3://lake/orders/") From 66552ebbd2068c2f7aed5ddcf1ab6743b939d001 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:54:04 +0100 Subject: [PATCH 05/21] release: 0.2.1 review fixes: CLI README, nightly gate accepts one .github-only commit, unsourced speedup claim removed Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .github/scripts/require-green-nightly.sh | 12 ++++++++++-- crates/krishiv-alloc/README.md | 2 +- crates/krishiv-cli/README.md | 7 ++++--- crates/krishiv-connect-server/README.md | 2 +- crates/krishiv-mcp/README.md | 2 +- crates/krishiv/README.md | 4 ++-- python/PYPI.md | 4 ++-- 7 files changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/scripts/require-green-nightly.sh b/.github/scripts/require-green-nightly.sh index f13548ef3..121dca910 100755 --- a/.github/scripts/require-green-nightly.sh +++ b/.github/scripts/require-green-nightly.sh @@ -33,7 +33,15 @@ status=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$run_sha" --jq . case "$status" in identical | ahead) echo "the scheduled run covers $GITHUB_SHA ($status)" ;; *) - echo "::error::the latest green scheduled run ($run_sha) does not include $GITHUB_SHA ($status); wait for the next nightly run" - exit 1 + # A publish tag (RELEASING.md "Publish tag"): exactly one commit that changes only files under .github/, + # directly on the commit the green run covers. + tag=$(gh api "repos/$GITHUB_REPOSITORY/compare/$run_sha...$GITHUB_SHA" --jq '[.status, .ahead_by, .behind_by, ([.files[].filename | select(startswith(".github/") | not)] | length)] | @tsv') + IFS=$'\t' read -r tag_status ahead behind outside <<<"$tag" + if [ "$tag_status" = ahead ] && [ "$ahead" = 1 ] && [ "$behind" = 0 ] && [ "$outside" = 0 ]; then + echo "$GITHUB_SHA is one .github-only commit on $run_sha, which the scheduled run covers" + else + echo "::error::the latest green scheduled run ($run_sha) does not include $GITHUB_SHA ($status); wait for the next nightly run" + exit 1 + fi ;; esac diff --git a/crates/krishiv-alloc/README.md b/crates/krishiv-alloc/README.md index d6073de2f..4da974391 100644 --- a/crates/krishiv-alloc/README.md +++ b/crates/krishiv-alloc/README.md @@ -1,7 +1,7 @@ # krishiv-alloc An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. -It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +It is published only because other Krishiv crates depend on it. Use the [`krishiv`](https://crates.io/crates/krishiv) crate instead; this crate's API is not stable and can change in any release. [Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-cli/README.md b/crates/krishiv-cli/README.md index 8eb5bb8c8..59ff58521 100644 --- a/crates/krishiv-cli/README.md +++ b/crates/krishiv-cli/README.md @@ -1,7 +1,8 @@ # krishiv-cli -An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. -It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) -crate instead; this crate's API is not stable and can change in any release. +The `krishiv` command-line binary of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames +without the JVM. Install it with `cargo install krishiv-cli`, or use the prebuilt binaries from the +[releases page](https://github.com/KrishivAI/krishiv-engine/releases). To use the engine from Rust code, depend on the +[`krishiv`](https://crates.io/crates/krishiv) crate instead. [Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-connect-server/README.md b/crates/krishiv-connect-server/README.md index 42a7d2400..cc51cb387 100644 --- a/crates/krishiv-connect-server/README.md +++ b/crates/krishiv-connect-server/README.md @@ -1,7 +1,7 @@ # krishiv-connect-server An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. -It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +It is published only because other Krishiv crates depend on it. Use the [`krishiv`](https://crates.io/crates/krishiv) crate instead; this crate's API is not stable and can change in any release. [Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv-mcp/README.md b/crates/krishiv-mcp/README.md index 95e009cad..10ccc780a 100644 --- a/crates/krishiv-mcp/README.md +++ b/crates/krishiv-mcp/README.md @@ -1,7 +1,7 @@ # krishiv-mcp An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. -It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +It is published only because other Krishiv crates depend on it. Use the [`krishiv`](https://crates.io/crates/krishiv) crate instead; this crate's API is not stable and can change in any release. [Website](https://krishiv.ai) · [Documentation](https://krishiv.ai/docs) · [Source](https://github.com/KrishivAI/krishiv-engine) · Apache-2.0 diff --git a/crates/krishiv/README.md b/crates/krishiv/README.md index 507a2a707..68b29b6fe 100644 --- a/crates/krishiv/README.md +++ b/crates/krishiv/README.md @@ -33,8 +33,8 @@ Actions are `async` and run on the session's own executor, so any runtime (or no ## Speed -On TPC-H at scale factor 100, Krishiv is **10.4x faster than Spark 4.2.0** (derived from TPC-H, not an official -TPC result). Method and numbers: . +Benchmark results against Spark 4.2.0 on TPC-H-derived queries (not official TPC results) are published with their +method and numbers at . ## Status diff --git a/python/PYPI.md b/python/PYPI.md index 0c0855afe..03247e03a 100644 --- a/python/PYPI.md +++ b/python/PYPI.md @@ -44,8 +44,8 @@ there is no stable API yet. ## Speed -On TPC-H at scale factor 100, Krishiv is **10.4x faster than Spark 4.2.0** (derived from TPC-H, not an official -TPC result). Method, queries and raw numbers: . +Benchmark results against Spark 4.2.0 on TPC-H-derived queries (not official TPC results) are published with their +method, queries and raw numbers at . ## Supported CPUs From 1677f774ed8bab50aec990500167f406d9d1cfa8 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:56:27 +0100 Subject: [PATCH 06/21] changelog: 751 compat fragment (pyarrow required) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- changelog.d/751.compat.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/751.compat.md diff --git a/changelog.d/751.compat.md b/changelog.d/751.compat.md new file mode 100644 index 000000000..d95433195 --- /dev/null +++ b/changelog.d/751.compat.md @@ -0,0 +1 @@ +`pip install krishiv` now installs `pyarrow>=16`, which was optional before; the 0.2.1 wheels and crates carry PyPI and crates.io metadata and READMEs. From 6c501a626659044bd14d1622049b9eaf410af385 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:09:43 +0100 Subject: [PATCH 07/21] Cargo.toml: taplo alignment after the workspace keywords and categories Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3b80bc1ca..2c010190e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ version = "0.2.1" edition = "2024" # MSRV = latest stable (1.99) minus two minor releases. rust-version = "1.97" -license = "Apache-2.0" # ADR 0033 +license = "Apache-2.0" # ADR 0033 repository = "https://github.com/KrishivAI/krishiv-engine" keywords = ["spark", "pyspark", "sql", "dataframe", "query-engine"] categories = ["database-implementations", "database"] From ab04ba9a15e08291e54647c3a0c4e3e6cf1eb45f Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:52:31 +0100 Subject: [PATCH 08/21] docs: the home page carries a release note, not an alpha box; install is from PyPI Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- docs/index.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index 53d1ee0fa..aaabfe7ad 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,13 +8,11 @@ Krishiv is a Rust-native, Apache Spark-compatible compute engine: batch SQL, Dat maintained views on one vectorized, Arrow-native engine, with no JVM. Keep the PySpark and Spark SQL you already write. -> **Alpha (`0.2.0-alpha`).** The embedded core runs: in-process batch SQL and DataFrames from Python. There is no -> stable API yet. Streaming, incremental views, Spark Connect and distributed execution are being built. +> Krishiv 0.2 is released on PyPI and crates.io. APIs can change before 1.0. ## Install -Nothing is on PyPI yet, so build the wheel from the repository (`maturin build` in `python/`) and install it. Once -released, `pip install "krishiv[pyarrow]"` is all it takes; the [Quickstart](get-started/quickstart.md) has the steps. +Install with `pip install "krishiv[pyarrow]"`; the [Quickstart](get-started/quickstart.md) has the steps. ## Where to start From 373ccfd1435ab7ae04d991a96ae55ff76a220969 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:03:17 +0100 Subject: [PATCH 09/21] RELEASING.md: the Docs step for a minor release (docs/0.X branch, version registration, deploy, check) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- RELEASING.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/RELEASING.md b/RELEASING.md index b1cce73ea..173ef4ea2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -183,6 +183,19 @@ Owner decision 2026-11-07 (default adopted): a minor release every **6 weeks**. keeps its dates and the next minor takes what is ready. - Breaking changes (docs/concepts/stability.md) go into a minor only, with the changelog "Breaking" heading. +## Docs (every minor release) + +Owner decision 2026-11-14: the live docs of each minor version are read from a branch `docs/0.X`, not from a tag, so a +docs fix for a released version needs no new release. After the tag `v0.X.0` is pushed and the release is verified: + +1. **Cut the branch** `docs/0.X` from the tag `v0.X.0` and protect it like `main` (PRs only, docs CI runs on it). + Docs-only fixes for that version are PRs to `docs/0.X`; patch releases (`v0.X.Z`) are merged into it. +2. **Register the version** in krishiv-internal `site/docs-plugin.json`: add + `engine/0.X = Krishiv Engine 0.X | docs @ docs/0.X`, mark it latest, and leave `engine/next` on `main` (staging only). +3. **Deploy** with `site/deploy.sh`. +4. **Check** the version switcher lists the new version as latest and that the previous version still serves its own + pages. + ## Before any release - `ui/scripts/sync-assets.sh` refreshes the committed UI assets From 3913ecb11b2b79e220e1f81bed206fb4b5ab4b59 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:24:45 +0100 Subject: [PATCH 10/21] krishiv-udf-sandbox: pin krishiv-common to =0.2.1 with the workspace Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- Cargo.lock | 2 +- crates/krishiv-udf-sandbox/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cbd7d14e..70c0432b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2835,7 +2835,7 @@ dependencies = [ [[package]] name = "krishiv-udf-sandbox" -version = "0.2.0" +version = "0.2.1" dependencies = [ "krishiv-common", "landlock", diff --git a/crates/krishiv-udf-sandbox/Cargo.toml b/crates/krishiv-udf-sandbox/Cargo.toml index 99bb81b3e..8ea93c290 100644 --- a/crates/krishiv-udf-sandbox/Cargo.toml +++ b/crates/krishiv-udf-sandbox/Cargo.toml @@ -12,7 +12,7 @@ publish = true workspace = true [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } thiserror.workspace = true [target.'cfg(target_os = "linux")'.dependencies] From 3d7fed7264e99d3c90eadbaa9f8fbb3a229db4c8 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:40:41 +0100 Subject: [PATCH 11/21] CHANGELOG: 0.2.1 also carries the acceptance-test fixes Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f95721e..c40af83b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Nothing yet. ## 0.2.1 -A patch release: packaging and documentation only, no engine changes. +A patch release: packaging, documentation and the fixes found by acceptance testing of 0.2.0. The fixes are listed from their changelog fragments when the release is assembled (`cargo xtask changelog assemble 0.2.1`). ### Changed From 43789d8c1231de48c35afca4eecbfe145724c530 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:46:56 +0100 Subject: [PATCH 12/21] python, wheels, docs: every wheel includes cloud object stores (s3://, gs://, az://); wheel test against MinIO; Data sources guide Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .github/workflows/wheels.yml | 11 ++++- changelog.d/751.fixed.md | 1 + crates/krishiv-python/Cargo.toml | 7 ++- docs/guides/_meta.json | 1 + docs/guides/data-sources.md | 79 +++++++++++++++++++++++++++++++ python/tests/test_cloud_stores.py | 61 ++++++++++++++++++++++++ 6 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 changelog.d/751.fixed.md create mode 100644 docs/guides/data-sources.md create mode 100644 python/tests/test_cloud_stores.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b17144110..adc68b2f8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -41,7 +41,7 @@ env: CARGO_INCREMENTAL: 0 # maturin build flags per flavour; `dist` is the LTO + stripped profile of the workspace Cargo.toml. ABI3_ARGS: --profile dist --locked - FT_ARGS: --profile dist --locked --no-default-features --features mimalloc + FT_ARGS: --profile dist --locked --no-default-features --features mimalloc,cloud # The shipped CPU levels (see README.md, "Supported CPUs"). X86_RUSTFLAGS: -C target-cpu=x86-64-v3 ARM_RUSTFLAGS: -C target-cpu=neoverse-n1 @@ -86,6 +86,15 @@ jobs: for w in dist/*.whl; do unzip -l "$w" | grep -q 'krishiv/_cpu_level$'; done - uses: $/.github/actions/setup-uv if: matrix.libc == 'manylinux' + # The cloud-store test reads and writes s3:// against MinIO: the bucket is a directory of its data dir. + - name: S3 emulator (MinIO) + if: matrix.libc == 'manylinux' + run: | + mkdir -p "$RUNNER_TEMP/minio/krishiv-test" + docker run -d --rm --name krishiv-wheel-minio -p 127.0.0.1:9000:9000 --user "$(id -u):$(id -g)" \ + -v "$RUNNER_TEMP/minio:/data" cgr.dev/chainguard/minio server /data + for _ in $(seq 1 60); do curl -s -o /dev/null --max-time 2 http://127.0.0.1:9000/minio/health/live && break; sleep 1; done + echo "KRISHIV_TEST_S3_ENDPOINT=http://127.0.0.1:9000" >> "$GITHUB_ENV" - name: Test (manylinux, on the runner) if: matrix.libc == 'manylinux' env: diff --git a/changelog.d/751.fixed.md b/changelog.d/751.fixed.md new file mode 100644 index 000000000..64439f094 --- /dev/null +++ b/changelog.d/751.fixed.md @@ -0,0 +1 @@ +The Python wheels now read and write `s3://`, `gs://` and `az://` URLs; the 0.2.0 wheels were built without cloud object stores and failed with `KRISHIV_UNSUPPORTED_FEATURE`. The new Data sources guide lists the credential, endpoint and region options. diff --git a/crates/krishiv-python/Cargo.toml b/crates/krishiv-python/Cargo.toml index 780a7b0d6..92b6f11c5 100644 --- a/crates/krishiv-python/Cargo.toml +++ b/crates/krishiv-python/Cargo.toml @@ -28,11 +28,14 @@ doctest = false [features] # Default wheel: one abi3 wheel for CPython >= 3.11. The free-threaded (cp314t) # build cannot use abi3, so it builds with -# `--no-default-features --features mimalloc`. -default = ["abi3-py311", "mimalloc"] +# `--no-default-features --features mimalloc,cloud`. +default = ["abi3-py311", "mimalloc", "cloud"] abi3-py311 = ["pyo3/abi3-py311"] # mimalloc is the global allocator of the extension module (P2-T012). mimalloc = ["dep:mimalloc"] +# Cloud object stores (s3://, gs://, az://) in every wheel: without it they fail with +# KRISHIV_UNSUPPORTED_FEATURE. 0.2.0 shipped without it. +cloud = ["krishiv-session/cloud"] [dependencies] arrow-array = { workspace = true, features = ["ffi"] } diff --git a/docs/guides/_meta.json b/docs/guides/_meta.json index a3f86e01c..03592f579 100644 --- a/docs/guides/_meta.json +++ b/docs/guides/_meta.json @@ -5,5 +5,6 @@ "check-compat": "Check a project with check-compat", "run-sql-from-the-command-line": "Run SQL from the command line", "tokio-console": "Task diagnostics with tokio-console", + "data-sources": "Data sources: files, S3, GCS and Azure", "notebooks-and-datasets": "Notebooks and sample data" } diff --git a/docs/guides/data-sources.md b/docs/guides/data-sources.md new file mode 100644 index 000000000..9fdc1d7e1 --- /dev/null +++ b/docs/guides/data-sources.md @@ -0,0 +1,79 @@ +--- +description: "Read and write Parquet, CSV and other files on local disk and on S3, Google Cloud Storage and Azure Blob Storage." +--- + +# Data sources + +Krishiv reads and writes files from your own disk and, from 0.2.1, from cloud object stores: Amazon S3 (`s3://`, `s3a://`), +Google Cloud Storage (`gs://`) and Azure Blob Storage (`az://`, `abfs://`, `abfss://`). The wheels from PyPI include the +object-store support; the 0.2.0 wheels did not, and failed with `KRISHIV_UNSUPPORTED_FEATURE` on these URLs. + +```python +import krishiv + +spark = krishiv.session() +spark.conf.set("fs.s3a.endpoint.region", "eu-west-1") +df = spark.read.parquet("s3://my-bucket/events/") +df.write.mode("overwrite").parquet("s3://my-bucket/summary/") +``` + +## Credentials and options + +Set the options on the session with `spark.conf.set`, or for one read or write with `.option(key, value)`. A per-read +option overrides the session value, and two reads of one bucket with different credentials never share a connection. +Keys match without regard to case. An option value is never printed: errors and `EXPLAIN` name the key only. + +When an option is not set, Krishiv falls back to the standard environment variables of each cloud (`AWS_ACCESS_KEY_ID`, +`AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, `GOOGLE_APPLICATION_CREDENTIALS`, `AZURE_STORAGE_ACCOUNT_NAME`, and so on). + +### Amazon S3 and S3-compatible stores + +| Option | Meaning | +|---|---| +| `fs.s3a.access.key`, `fs.s3a.secret.key` | access key id and secret | +| `fs.s3a.session.token` | session token, for temporary credentials | +| `fs.s3a.endpoint.region` (or `fs.s3a.region`) | region; the default is `us-east-1` | +| `fs.s3a.endpoint` | endpoint URL for MinIO, LocalStack, Ceph and similar; `http://` is allowed | +| `fs.s3a.path.style.access` | `true` forces `https://host/bucket/key` requests; already the default for `http://` and `localhost` endpoints | + +### Google Cloud Storage + +| Option | Meaning | +|---|---| +| `fs.gs.auth.service.account.json.keyfile` | path to a service-account JSON file | +| `fs.gs.auth.service.account.json` | the service-account JSON itself | +| `fs.gs.storage.root.url` (or `fs.gs.endpoint`) | endpoint URL for an emulator | + +### Azure Blob Storage + +| Option | Meaning | +|---|---| +| `fs.azure.account.key..dfs.core.windows.net` | storage account key (the `.blob.` form works too) | +| `fs.azure.sas...dfs.core.windows.net` | shared access signature for one container | +| `fs.azure.account.name` | account name, when the URL is `az://container/path` and does not carry it | +| `fs.azure.endpoint` | endpoint URL for Azurite (`http://127.0.0.1:10000/`) | + +## Try it without a cloud account + +Run MinIO locally, create a bucket as a directory of its data folder, and point `fs.s3a.endpoint` at it: + +```bash +mkdir -p data/my-bucket +docker run --rm -p 9000:9000 -v "$PWD/data:/data" quay.io/minio/minio server /data +``` + +```python +for key, value in { + "fs.s3a.endpoint": "http://127.0.0.1:9000", + "fs.s3a.access.key": "minioadmin", + "fs.s3a.secret.key": "minioadmin", +}.items(): + spark.conf.set(key, value) +spark.read.parquet("s3://my-bucket/events/").show() +``` + +## On a shared server + +In server mode with authentication on, reads and writes by path are confined to the prefixes in +`krishiv.io.objectStore.allowedPrefixes` and `krishiv.io.local.allowedRoots`; see the +[configuration reference](../reference/config/index.md). diff --git a/python/tests/test_cloud_stores.py b/python/tests/test_cloud_stores.py new file mode 100644 index 000000000..e5353c468 --- /dev/null +++ b/python/tests/test_cloud_stores.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Cloud object stores are built into the wheel (0.2.0 shipped without them: ``s3://``, ``gs://`` and ``az://`` +failed with KRISHIV_UNSUPPORTED_FEATURE). + +The offline test runs everywhere: a store URL must get past "not built in" and fail on the network instead. The round +trip runs when ``KRISHIV_TEST_S3_ENDPOINT`` names an S3 emulator with a bucket ``krishiv-test`` (MinIO, credentials +``minioadmin``/``minioadmin``; ``tests/scripts/object_store_emulators.sh`` and the wheels workflow start one).""" + +from __future__ import annotations + +import os +import uuid +from typing import Any + +import pytest + +from krishiv.sql import SparkSession + +pa = pytest.importorskip("pyarrow") + +ENDPOINT = os.environ.get("KRISHIV_TEST_S3_ENDPOINT") + + +@pytest.fixture +def spark() -> SparkSession: + return SparkSession.builder.create() + + +@pytest.mark.parametrize( + "url", ["s3://krishiv-test/x.parquet", "gs://krishiv-test/x.parquet", "az://krishiv-test/x.parquet"] +) +def test_cloud_urls_are_built_into_the_wheel(spark: SparkSession, url: str) -> None: + # Point every store at a closed local port so nothing leaves the machine and the failure is quick. + for key, value in { + "fs.s3a.endpoint": "http://127.0.0.1:9", + "fs.s3a.access.key": "k", + "fs.s3a.secret.key": "s", + "fs.s3a.endpoint.region": "us-east-1", + "fs.s3a.connection.maximum": "1", + }.items(): + spark.conf.set(key, value) + with pytest.raises(Exception) as err: # noqa: PT011 - the class depends on the store; the text is what matters + spark.read.parquet(url).collect() + assert "KRISHIV_UNSUPPORTED_FEATURE" not in str(err.value), err.value + + +@pytest.mark.skipif(not ENDPOINT, reason="KRISHIV_TEST_S3_ENDPOINT is not set") +def test_s3_round_trip_against_an_emulator(spark: SparkSession) -> None: + options: dict[str, Any] = { + "fs.s3a.endpoint": ENDPOINT, + "fs.s3a.access.key": "minioadmin", + "fs.s3a.secret.key": "minioadmin", + "fs.s3a.path.style.access": "true", + "fs.s3a.endpoint.region": "us-east-1", + } + for key, value in options.items(): + spark.conf.set(key, value) + url = f"s3://krishiv-test/wheel-{uuid.uuid4().hex}" + spark.createDataFrame(pa.table({"id": [3, 1, 2], "name": ["c", None, "b"]})).write.parquet(url) + rows = sorted((r["id"], r["name"]) for r in spark.read.parquet(url).collect()) + assert rows == [(1, None), (2, "b"), (3, "c")] From 891906d8f0c29531d176a2c7fe8f132ca2162287 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:03:40 +0100 Subject: [PATCH 13/21] ci: the free-threaded CI wheel builds with the cloud feature too Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- python/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef025b2c2..89eba1190 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -665,7 +665,7 @@ jobs: coverage: true - name: cp314t python: "3.14t" - maturin_args: "--no-default-features --features mimalloc" + maturin_args: "--no-default-features --features mimalloc,cloud" wheel_tag: cp314t coverage: false name: wheels (${{ matrix.name }}) diff --git a/python/README.md b/python/README.md index 16d0a65a1..1aa092eed 100644 --- a/python/README.md +++ b/python/README.md @@ -105,7 +105,7 @@ non-abi3 build (disable the default `abi3-py311` crate feature but keep `mimallo cd python uv venv --python 3.14t .venv-314t && uv pip install --python .venv-314t/bin/python maturin pytest . .venv-314t/bin/activate -maturin develop --no-default-features --features mimalloc # or: maturin build ... +maturin develop --no-default-features --features mimalloc,cloud # or: maturin build ... # (mimalloc is a default feature; keep it when turning the defaults off) ``` From c92d5fb04faa70de57a1886dbf24bb711ebb371d Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:09:33 +0100 Subject: [PATCH 14/21] ci: the wheel test installs the wheel with --no-deps so a target without a pyarrow wheel (musllinux) does not build pyarrow from source Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .github/scripts/wheel-test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/wheel-test.sh b/.github/scripts/wheel-test.sh index ffbffeb4d..4a542255a 100755 --- a/.github/scripts/wheel-test.sh +++ b/.github/scripts/wheel-test.sh @@ -21,7 +21,9 @@ work=$(mktemp -d) "$python" -m venv "$work/venv" py="$work/venv/bin/python" "$py" -m pip install --quiet --upgrade pip -"$py" -m pip install --quiet "$dist"/*.whl pytest +# --no-deps: the wheel requires pyarrow, which has no musllinux wheel and would build from source (an hour); +# pyarrow comes from the binary-only loop below, and the smoke falls back to count() without it. +"$py" -m pip install --quiet --no-deps "$dist"/*.whl pytest # Versions come from python/ci-interop-constraints.txt, the same pins ci.yml # and nightly.yml use, so every wheel leg tests against one known set. # pandas and numpy: the pandas_udf tests and the PySpark UDF conformance tests (T059 unit 3a); a missing wheel skips them. From b82d7f69bd8418eaf6c8b0b5eb83e0536aa1b9cb Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:24:14 +0100 Subject: [PATCH 15/21] ci: the wheel test installs pytest with its dependencies; --no-deps is for the Krishiv wheel only Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .github/scripts/wheel-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/wheel-test.sh b/.github/scripts/wheel-test.sh index 4a542255a..291ca7197 100755 --- a/.github/scripts/wheel-test.sh +++ b/.github/scripts/wheel-test.sh @@ -23,7 +23,8 @@ py="$work/venv/bin/python" "$py" -m pip install --quiet --upgrade pip # --no-deps: the wheel requires pyarrow, which has no musllinux wheel and would build from source (an hour); # pyarrow comes from the binary-only loop below, and the smoke falls back to count() without it. -"$py" -m pip install --quiet --no-deps "$dist"/*.whl pytest +"$py" -m pip install --quiet --no-deps "$dist"/*.whl +"$py" -m pip install --quiet pytest # Versions come from python/ci-interop-constraints.txt, the same pins ci.yml # and nightly.yml use, so every wheel leg tests against one known set. # pandas and numpy: the pandas_udf tests and the PySpark UDF conformance tests (T059 unit 3a); a missing wheel skips them. From 2ad9d1c7e1eb591d25ff002c762ce63ec717ce3a Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:50:07 +0100 Subject: [PATCH 16/21] krishiv, cli: the telemetry-otlp feature forwards to krishiv-session (RS-15) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- changelog.d/751.added.md | 1 + crates/krishiv-cli/Cargo.toml | 2 ++ crates/krishiv/Cargo.toml | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 changelog.d/751.added.md diff --git a/changelog.d/751.added.md b/changelog.d/751.added.md new file mode 100644 index 000000000..1eb9b837f --- /dev/null +++ b/changelog.d/751.added.md @@ -0,0 +1 @@ +The `krishiv` and `krishiv-cli` crates now have a `telemetry-otlp` feature that forwards to `krishiv-session/telemetry-otlp` (OTLP export of the query spans and engine metrics); 0.2.0 named it in the changelog but only `krishiv-session` had it. diff --git a/crates/krishiv-cli/Cargo.toml b/crates/krishiv-cli/Cargo.toml index 3c61a9bb9..ccb0ee4a0 100644 --- a/crates/krishiv-cli/Cargo.toml +++ b/crates/krishiv-cli/Cargo.toml @@ -82,6 +82,8 @@ mimalloc = ["krishiv/mimalloc", "dep:mimalloc", "dep:krishiv-alloc", "dep:krishi tokio-console = ["krishiv/tokio-console"] # JSON logs on stderr with `KRISHIV_LOG=json` (T116). telemetry-logs = ["krishiv/telemetry-logs", "dep:krishiv-session"] +# OTLP export of the query spans and engine metrics. +telemetry-otlp = ["krishiv/telemetry-otlp", "dep:krishiv-session"] # Cloud object stores (S3, GCS, Azure). Opt-in so the default build stays # inside the dependency and binary-size budgets. cloud = ["krishiv/cloud"] diff --git a/crates/krishiv/Cargo.toml b/crates/krishiv/Cargo.toml index 29508b190..fa1e7f9e9 100644 --- a/crates/krishiv/Cargo.toml +++ b/crates/krishiv/Cargo.toml @@ -57,4 +57,6 @@ tokio-console = ["dep:krishiv-exec", "krishiv-exec/tokio-console"] # JSON log records with query, session and trace ids (T116), installed by the CLI # when `KRISHIV_LOG=json`; embedders call `krishiv_session::logging` themselves. telemetry-logs = ["krishiv-session/telemetry-logs"] +# OTLP export of the query spans and engine metrics (`krishiv_session::otlp`), off by default. +telemetry-otlp = ["krishiv-session/telemetry-otlp"] full = ["kafka", "iceberg", "delta", "cdc", "connect", "flight-sql", "mcp", "python", "wasm-udf", "ui", "celeborn", "mimalloc", "cloud"] From b0f2081c054f556185695dbfbec0db999b89a592 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:53:58 +0100 Subject: [PATCH 17/21] python: the cloud-store wheel test also covers https:// and hf:// (PY-031, PY-032) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- python/tests/test_cloud_stores.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/tests/test_cloud_stores.py b/python/tests/test_cloud_stores.py index e5353c468..469784707 100644 --- a/python/tests/test_cloud_stores.py +++ b/python/tests/test_cloud_stores.py @@ -27,7 +27,14 @@ def spark() -> SparkSession: @pytest.mark.parametrize( - "url", ["s3://krishiv-test/x.parquet", "gs://krishiv-test/x.parquet", "az://krishiv-test/x.parquet"] + "url", + [ + "s3://krishiv-test/x.parquet", + "gs://krishiv-test/x.parquet", + "az://krishiv-test/x.parquet", + "https://127.0.0.1:9/x.parquet", + "hf://datasets/krishiv-test/x/x.parquet", + ], ) def test_cloud_urls_are_built_into_the_wheel(spark: SparkSession, url: str) -> None: # Point every store at a closed local port so nothing leaves the machine and the failure is quick. @@ -42,6 +49,7 @@ def test_cloud_urls_are_built_into_the_wheel(spark: SparkSession, url: str) -> N with pytest.raises(Exception) as err: # noqa: PT011 - the class depends on the store; the text is what matters spark.read.parquet(url).collect() assert "KRISHIV_UNSUPPORTED_FEATURE" not in str(err.value), err.value + assert "hadoop" not in str(err.value).lower(), err.value # PY-031: https:// once failed naming a Hadoop class @pytest.mark.skipif(not ENDPOINT, reason="KRISHIV_TEST_S3_ENDPOINT is not set") From 888d741c2f4c587392e0d032a958c7c53b80abb1 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:54:31 +0100 Subject: [PATCH 18/21] packaging fixes: issue ids in the changelog fragments (PY-021, PY-031, PY-032, SQL-1, PY-022, PY-006, RS-3, RS-15) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- changelog.d/751.added.md | 2 +- changelog.d/751.compat.md | 2 +- changelog.d/751.fixed.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/751.added.md b/changelog.d/751.added.md index 1eb9b837f..add3f534b 100644 --- a/changelog.d/751.added.md +++ b/changelog.d/751.added.md @@ -1 +1 @@ -The `krishiv` and `krishiv-cli` crates now have a `telemetry-otlp` feature that forwards to `krishiv-session/telemetry-otlp` (OTLP export of the query spans and engine metrics); 0.2.0 named it in the changelog but only `krishiv-session` had it. +(RS-15) The `krishiv` and `krishiv-cli` crates now have a `telemetry-otlp` feature that forwards to `krishiv-session/telemetry-otlp` (OTLP export of the query spans and engine metrics); 0.2.0 named it in the changelog but only `krishiv-session` had it. diff --git a/changelog.d/751.compat.md b/changelog.d/751.compat.md index d95433195..ec1ab9a7c 100644 --- a/changelog.d/751.compat.md +++ b/changelog.d/751.compat.md @@ -1 +1 @@ -`pip install krishiv` now installs `pyarrow>=16`, which was optional before; the 0.2.1 wheels and crates carry PyPI and crates.io metadata and READMEs. +(SQL-1, PY-022, PY-006, RS-3) `pip install krishiv` now installs `pyarrow>=16`, which was optional before; the 0.2.1 wheels and crates carry PyPI and crates.io metadata and READMEs. diff --git a/changelog.d/751.fixed.md b/changelog.d/751.fixed.md index 64439f094..0e8338fb6 100644 --- a/changelog.d/751.fixed.md +++ b/changelog.d/751.fixed.md @@ -1 +1 @@ -The Python wheels now read and write `s3://`, `gs://` and `az://` URLs; the 0.2.0 wheels were built without cloud object stores and failed with `KRISHIV_UNSUPPORTED_FEATURE`. The new Data sources guide lists the credential, endpoint and region options. +(PY-021, PY-031, PY-032) The Python wheels now read and write `s3://`, `gs://` and `az://` URLs; the 0.2.0 wheels were built without cloud object stores and failed with `KRISHIV_UNSUPPORTED_FEATURE`. The new Data sources guide lists the credential, endpoint and region options. From b881e7218c3a4a24d691643e960a80b7365606f3 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:19:05 +0100 Subject: [PATCH 19/21] cli, session, python: telemetry-otlp in feature_states; session dev-dep pinned to 0.2.1; https is Spark's UNSUPPORTED_FILE_SYSTEM, not a wheel test case Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-cli/src/bundle.rs | 4 ++++ crates/krishiv-session/Cargo.toml | 2 +- python/tests/test_cloud_stores.py | 2 -- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/krishiv-cli/src/bundle.rs b/crates/krishiv-cli/src/bundle.rs index 508aff9a6..062cd82b4 100644 --- a/crates/krishiv-cli/src/bundle.rs +++ b/crates/krishiv-cli/src/bundle.rs @@ -518,6 +518,7 @@ pub(crate) fn feature_states() -> Vec<(&'static str, bool)> { ("celeborn", cfg!(feature = "celeborn")), ("tokio-console", cfg!(feature = "tokio-console")), ("telemetry-logs", cfg!(feature = "telemetry-logs")), + ("telemetry-otlp", cfg!(feature = "telemetry-otlp")), ("full", cfg!(feature = "full")), ] } @@ -572,6 +573,9 @@ fn enabled_features() -> Vec<&'static str> { if cfg!(feature = "telemetry-logs") { out.push("telemetry-logs"); } + if cfg!(feature = "telemetry-otlp") { + out.push("telemetry-otlp"); + } if cfg!(feature = "full") { out.push("full"); } diff --git a/crates/krishiv-session/Cargo.toml b/crates/krishiv-session/Cargo.toml index 49378bdfd..487978a16 100644 --- a/crates/krishiv-session/Cargo.toml +++ b/crates/krishiv-session/Cargo.toml @@ -61,7 +61,7 @@ object_store = { workspace = true, features = ["fs"] } [dev-dependencies] bytes.workspace = true -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0", features = ["testing"] } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1", features = ["testing"] } arrow-select.workspace = true futures = { workspace = true, features = ["alloc", "executor"] } insta.workspace = true diff --git a/python/tests/test_cloud_stores.py b/python/tests/test_cloud_stores.py index 469784707..31d1342b4 100644 --- a/python/tests/test_cloud_stores.py +++ b/python/tests/test_cloud_stores.py @@ -32,7 +32,6 @@ def spark() -> SparkSession: "s3://krishiv-test/x.parquet", "gs://krishiv-test/x.parquet", "az://krishiv-test/x.parquet", - "https://127.0.0.1:9/x.parquet", "hf://datasets/krishiv-test/x/x.parquet", ], ) @@ -49,7 +48,6 @@ def test_cloud_urls_are_built_into_the_wheel(spark: SparkSession, url: str) -> N with pytest.raises(Exception) as err: # noqa: PT011 - the class depends on the store; the text is what matters spark.read.parquet(url).collect() assert "KRISHIV_UNSUPPORTED_FEATURE" not in str(err.value), err.value - assert "hadoop" not in str(err.value).lower(), err.value # PY-031: https:// once failed naming a Hadoop class @pytest.mark.skipif(not ENDPOINT, reason="KRISHIV_TEST_S3_ENDPOINT is not set") From 4b209a06a5cdd6cbefe76332a54bf3aa6ea6e01d Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:41:56 +0100 Subject: [PATCH 20/21] cli: enabled_features derives from feature_states (one list, no uncovered cfg branch) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- crates/krishiv-cli/src/bundle.rs | 59 +++----------------------------- 1 file changed, 4 insertions(+), 55 deletions(-) diff --git a/crates/krishiv-cli/src/bundle.rs b/crates/krishiv-cli/src/bundle.rs index 062cd82b4..3f5361c10 100644 --- a/crates/krishiv-cli/src/bundle.rs +++ b/crates/krishiv-cli/src/bundle.rs @@ -524,61 +524,10 @@ pub(crate) fn feature_states() -> Vec<(&'static str, bool)> { } fn enabled_features() -> Vec<&'static str> { - let mut out = Vec::new(); - if cfg!(feature = "mimalloc") { - out.push("mimalloc"); - } - if cfg!(feature = "server") { - out.push("server"); - } - if cfg!(feature = "ui") { - out.push("ui"); - } - if cfg!(feature = "kafka") { - out.push("kafka"); - } - if cfg!(feature = "iceberg") { - out.push("iceberg"); - } - if cfg!(feature = "delta") { - out.push("delta"); - } - if cfg!(feature = "cdc") { - out.push("cdc"); - } - if cfg!(feature = "connect") { - out.push("connect"); - } - if cfg!(feature = "flight-sql") { - out.push("flight-sql"); - } - if cfg!(feature = "mcp") { - out.push("mcp"); - } - if cfg!(feature = "python") { - out.push("python"); - } - if cfg!(feature = "collations-icu") { - out.push("collations-icu"); - } - if cfg!(feature = "wasm-udf") { - out.push("wasm-udf"); - } - if cfg!(feature = "celeborn") { - out.push("celeborn"); - } - if cfg!(feature = "tokio-console") { - out.push("tokio-console"); - } - if cfg!(feature = "telemetry-logs") { - out.push("telemetry-logs"); - } - if cfg!(feature = "telemetry-otlp") { - out.push("telemetry-otlp"); - } - if cfg!(feature = "full") { - out.push("full"); - } + let mut out: Vec<&'static str> = feature_states() + .into_iter() + .filter_map(|(name, on)| on.then_some(name)) + .collect(); out.sort_unstable(); out } From 1685294a35c27ffbc978c63e17af44738b85c533 Mon Sep 17 00:00:00 2001 From: gopal tathe <44522021+techgopal@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:06:06 +0100 Subject: [PATCH 21/21] ci, python: the crate-upload retry falls back to ten minutes without a retry date; the offline cloud-store test runs without pyarrow (review of #751) Signed-off-by: gopal tathe <44522021+techgopal@users.noreply.github.com> --- .github/workflows/publish-crates.yml | 3 ++- python/tests/test_cloud_stores.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 658cf305a..bb9462c8b 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -80,7 +80,8 @@ jobs: grep -q '429 Too Many Requests' "$log" || exit 1 [ "$attempt" -lt 40 ] || exit 1 after=$(grep -oE 'try again after [^.]+GMT' "$log" | head -1 | sed 's/try again after //') - wait=$(( $( date -d "$after" +%s 2>/dev/null || echo $(( $(date +%s) + 600 )) ) - $(date +%s) + 20 )) + # An empty or unparsable date (`date -d ""` is midnight today) falls back to ten minutes. + if [ -n "$after" ] && until=$(date -d "$after" +%s 2>/dev/null); then wait=$(( until - $(date +%s) + 20 )); else wait=600; fi [ "$wait" -gt 20 ] || wait=60 echo "rate limited on $crate (attempt $attempt): waiting ${wait}s until $after" sleep "$wait" diff --git a/python/tests/test_cloud_stores.py b/python/tests/test_cloud_stores.py index 31d1342b4..b9663eea4 100644 --- a/python/tests/test_cloud_stores.py +++ b/python/tests/test_cloud_stores.py @@ -16,8 +16,6 @@ from krishiv.sql import SparkSession -pa = pytest.importorskip("pyarrow") - ENDPOINT = os.environ.get("KRISHIV_TEST_S3_ENDPOINT") @@ -52,6 +50,7 @@ def test_cloud_urls_are_built_into_the_wheel(spark: SparkSession, url: str) -> N @pytest.mark.skipif(not ENDPOINT, reason="KRISHIV_TEST_S3_ENDPOINT is not set") def test_s3_round_trip_against_an_emulator(spark: SparkSession) -> None: + pa = pytest.importorskip("pyarrow") # the offline test above must still run where pyarrow is absent (musllinux) options: dict[str, Any] = { "fs.s3a.endpoint": ENDPOINT, "fs.s3a.access.key": "minioadmin",