diff --git a/.github/scripts/require-green-nightly.sh b/.github/scripts/require-green-nightly.sh index f13548ef3..121dca910 100755 --- a/.github/scripts/require-green-nightly.sh +++ b/.github/scripts/require-green-nightly.sh @@ -33,7 +33,15 @@ status=$(gh api "repos/$GITHUB_REPOSITORY/compare/$GITHUB_SHA...$run_sha" --jq . case "$status" in identical | ahead) echo "the scheduled run covers $GITHUB_SHA ($status)" ;; *) - echo "::error::the latest green scheduled run ($run_sha) does not include $GITHUB_SHA ($status); wait for the next nightly run" - exit 1 + # A publish tag (RELEASING.md "Publish tag"): exactly one commit that changes only files under .github/, + # directly on the commit the green run covers. + tag=$(gh api "repos/$GITHUB_REPOSITORY/compare/$run_sha...$GITHUB_SHA" --jq '[.status, .ahead_by, .behind_by, ([.files[].filename | select(startswith(".github/") | not)] | length)] | @tsv') + IFS=$'\t' read -r tag_status ahead behind outside <<<"$tag" + if [ "$tag_status" = ahead ] && [ "$ahead" = 1 ] && [ "$behind" = 0 ] && [ "$outside" = 0 ]; then + echo "$GITHUB_SHA is one .github-only commit on $run_sha, which the scheduled run covers" + else + echo "::error::the latest green scheduled run ($run_sha) does not include $GITHUB_SHA ($status); wait for the next nightly run" + exit 1 + fi ;; esac diff --git a/.github/scripts/wheel-test.sh b/.github/scripts/wheel-test.sh index ffbffeb4d..291ca7197 100755 --- a/.github/scripts/wheel-test.sh +++ b/.github/scripts/wheel-test.sh @@ -21,7 +21,10 @@ work=$(mktemp -d) "$python" -m venv "$work/venv" py="$work/venv/bin/python" "$py" -m pip install --quiet --upgrade pip -"$py" -m pip install --quiet "$dist"/*.whl pytest +# --no-deps: the wheel requires pyarrow, which has no musllinux wheel and would build from source (an hour); +# pyarrow comes from the binary-only loop below, and the smoke falls back to count() without it. +"$py" -m pip install --quiet --no-deps "$dist"/*.whl +"$py" -m pip install --quiet pytest # Versions come from python/ci-interop-constraints.txt, the same pins ci.yml # and nightly.yml use, so every wheel leg tests against one known set. # pandas and numpy: the pandas_udf tests and the PySpark UDF conformance tests (T059 unit 3a); a missing wheel skips them. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39887e699..96bd89500 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -681,7 +681,7 @@ jobs: coverage: true - name: cp314t python: "3.14t" - maturin_args: "--no-default-features --features mimalloc" + maturin_args: "--no-default-features --features mimalloc,cloud" wheel_tag: cp314t coverage: false name: wheels (${{ matrix.name }}) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index be69498e6..7dc3fa14b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,8 +9,6 @@ on: schedule: # Off the hour: the top of the hour is when scheduled runs are delayed or dropped most. - cron: "23 22 * * *" - # One-off extra runs on 2026-11-13 for the v0.2.0 release (binaries.yml and release.yml still need a green scheduled nightly; publishing no longer does); remove after. - - cron: "17 8,10 13 11 *" workflow_dispatch: permissions: diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 1dc354cb2..bb9462c8b 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -40,7 +40,7 @@ jobs: needs: ci if: github.repository == 'KrishivAI/krishiv-engine' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 330 environment: name: crates-io url: https://crates.io/crates/krishiv @@ -72,6 +72,19 @@ jobs: continue fi echo "::group::$crate $version" - cargo publish -p "$crate" --locked # zizmor: ignore[use-trusted-publishing] scoped token in the gated crates-io environment + # crates.io limits how fast new crate names are created (429 "try again after "): wait and retry. + for attempt in $(seq 1 40); do + log="$RUNNER_TEMP/publish-$crate.log" + if cargo publish -p "$crate" --locked >"$log" 2>&1; then cat "$log"; break; fi # zizmor: ignore[use-trusted-publishing] scoped token in the gated crates-io environment + cat "$log" + grep -q '429 Too Many Requests' "$log" || exit 1 + [ "$attempt" -lt 40 ] || exit 1 + after=$(grep -oE 'try again after [^.]+GMT' "$log" | head -1 | sed 's/try again after //') + # An empty or unparsable date (`date -d ""` is midnight today) falls back to ten minutes. + if [ -n "$after" ] && until=$(date -d "$after" +%s 2>/dev/null); then wait=$(( until - $(date +%s) + 20 )); else wait=600; fi + [ "$wait" -gt 20 ] || wait=60 + echo "rate limited on $crate (attempt $attempt): waiting ${wait}s until $after" + sleep "$wait" + done echo "::endgroup::" done < .github/scripts/publish-crates.txt diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b17144110..adc68b2f8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -41,7 +41,7 @@ env: CARGO_INCREMENTAL: 0 # maturin build flags per flavour; `dist` is the LTO + stripped profile of the workspace Cargo.toml. ABI3_ARGS: --profile dist --locked - FT_ARGS: --profile dist --locked --no-default-features --features mimalloc + FT_ARGS: --profile dist --locked --no-default-features --features mimalloc,cloud # The shipped CPU levels (see README.md, "Supported CPUs"). X86_RUSTFLAGS: -C target-cpu=x86-64-v3 ARM_RUSTFLAGS: -C target-cpu=neoverse-n1 @@ -86,6 +86,15 @@ jobs: for w in dist/*.whl; do unzip -l "$w" | grep -q 'krishiv/_cpu_level$'; done - uses: $/.github/actions/setup-uv if: matrix.libc == 'manylinux' + # The cloud-store test reads and writes s3:// against MinIO: the bucket is a directory of its data dir. + - name: S3 emulator (MinIO) + if: matrix.libc == 'manylinux' + run: | + mkdir -p "$RUNNER_TEMP/minio/krishiv-test" + docker run -d --rm --name krishiv-wheel-minio -p 127.0.0.1:9000:9000 --user "$(id -u):$(id -g)" \ + -v "$RUNNER_TEMP/minio:/data" cgr.dev/chainguard/minio server /data + for _ in $(seq 1 60); do curl -s -o /dev/null --max-time 2 http://127.0.0.1:9000/minio/health/live && break; sleep 1; done + echo "KRISHIV_TEST_S3_ENDPOINT=http://127.0.0.1:9000" >> "$GITHUB_ENV" - name: Test (manylinux, on the runner) if: matrix.libc == 'manylinux' env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a4ce75ee..c40af83b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,20 @@ Later entries are generated by release-plz from Conventional Commits (`release-p Nothing yet. +## 0.2.1 + +A patch release: packaging, documentation and the fixes found by acceptance testing of 0.2.0. The fixes are listed from their changelog fragments when the release is assembled (`cargo xtask changelog assemble 0.2.1`). + +### Changed + +- **`pyarrow>=16` is now a required dependency** of the Python package, so a bare `pip install krishiv` can `collect()` and + `show()`. `krishiv[pyarrow]` still works as an alias. +- The PyPI page (`krishiv` and `krishiv-compat`) now has a description (`python/PYPI.md`), keywords, classifiers and project URLs. +- The crates.io pages have READMEs: a full one for `krishiv` (its example runs as a doc test) and a short pointer to `krishiv` + for every internal crate; keywords and categories include `spark`. +- The crates.io publish workflow waits out the registry's 429 rate limit for new crate names and retries; the one-off + nightly cron is gone; `RELEASING.md` documents publishing from a tag. + ## 0.2.0 The first release is the embedded core: the engine runs in your process, from Python or Rust. diff --git a/Cargo.lock b/Cargo.lock index 67ad3cc06..1b5e4cb32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2287,7 +2287,7 @@ dependencies = [ [[package]] name = "krishiv" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2301,7 +2301,7 @@ dependencies = [ [[package]] name = "krishiv-alloc" -version = "0.2.0" +version = "0.2.1" dependencies = [ "krishiv-common", "libmimalloc-sys", @@ -2309,7 +2309,7 @@ dependencies = [ [[package]] name = "krishiv-bench-runner" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-row", @@ -2331,7 +2331,7 @@ dependencies = [ [[package]] name = "krishiv-catalogs" -version = "0.2.0" +version = "0.2.1" dependencies = [ "krishiv-common", "krishiv-plan", @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "krishiv-cli" -version = "0.2.0" +version = "0.2.1" dependencies = [ "axum", "clap", @@ -2369,7 +2369,7 @@ dependencies = [ [[package]] name = "krishiv-common" -version = "0.2.0" +version = "0.2.1" dependencies = [ "codspeed-criterion-compat", "serde_json", @@ -2379,7 +2379,7 @@ dependencies = [ [[package]] name = "krishiv-connect" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2395,7 +2395,7 @@ dependencies = [ [[package]] name = "krishiv-connect-server" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-ipc", @@ -2424,15 +2424,15 @@ dependencies = [ [[package]] name = "krishiv-connector-sdk" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-connectors-cdc" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-connectors-delta" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-arith", "arrow-array", @@ -2463,15 +2463,15 @@ dependencies = [ [[package]] name = "krishiv-connectors-iceberg" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-connectors-kafka" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-exec" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2510,11 +2510,11 @@ dependencies = [ [[package]] name = "krishiv-flight-sql" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-functions" -version = "0.2.0" +version = "0.2.1" dependencies = [ "aes", "aes-gcm", @@ -2536,7 +2536,7 @@ dependencies = [ [[package]] name = "krishiv-io" -version = "0.2.0" +version = "0.2.1" dependencies = [ "apache-avro", "arrow-array", @@ -2573,7 +2573,7 @@ dependencies = [ [[package]] name = "krishiv-kernels" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2600,7 +2600,7 @@ dependencies = [ [[package]] name = "krishiv-mcp" -version = "0.2.0" +version = "0.2.1" dependencies = [ "futures", "krishiv-common", @@ -2615,7 +2615,7 @@ dependencies = [ [[package]] name = "krishiv-memory" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2636,7 +2636,7 @@ dependencies = [ [[package]] name = "krishiv-optimizer" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2655,7 +2655,7 @@ dependencies = [ [[package]] name = "krishiv-plan" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2675,7 +2675,7 @@ dependencies = [ [[package]] name = "krishiv-plan-codec" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-ipc", "arrow-schema", @@ -2695,7 +2695,7 @@ dependencies = [ [[package]] name = "krishiv-python" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2714,7 +2714,7 @@ dependencies = [ [[package]] name = "krishiv-scheduler" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2736,7 +2736,7 @@ dependencies = [ [[package]] name = "krishiv-server" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2755,7 +2755,7 @@ dependencies = [ [[package]] name = "krishiv-session" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2803,7 +2803,7 @@ dependencies = [ [[package]] name = "krishiv-shuffle" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-ipc", @@ -2818,7 +2818,7 @@ dependencies = [ [[package]] name = "krishiv-sim" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-schema", @@ -2842,7 +2842,7 @@ dependencies = [ [[package]] name = "krishiv-sql" -version = "0.2.0" +version = "0.2.1" dependencies = [ "codspeed-criterion-compat", "insta", @@ -2856,11 +2856,11 @@ dependencies = [ [[package]] name = "krishiv-state" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-store" -version = "0.2.0" +version = "0.2.1" dependencies = [ "async-trait", "bytes", @@ -2873,11 +2873,11 @@ dependencies = [ [[package]] name = "krishiv-stream" -version = "0.2.0" +version = "0.2.1" [[package]] name = "krishiv-testkit" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2911,7 +2911,7 @@ dependencies = [ [[package]] name = "krishiv-types" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -2925,7 +2925,7 @@ dependencies = [ [[package]] name = "krishiv-udf-pool" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "arrow-ipc", @@ -2944,7 +2944,7 @@ dependencies = [ [[package]] name = "krishiv-udf-sandbox" -version = "0.2.0" +version = "0.2.1" dependencies = [ "krishiv-common", "landlock", @@ -2957,7 +2957,7 @@ dependencies = [ [[package]] name = "krishiv-udf-sdk" -version = "0.2.0" +version = "0.2.1" dependencies = [ "arrow-array", "krishiv-types", @@ -2965,7 +2965,7 @@ dependencies = [ [[package]] name = "krishiv-ui-assets" -version = "0.2.0" +version = "0.2.1" dependencies = [ "rust-embed", ] @@ -5659,7 +5659,7 @@ checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "xtask" -version = "0.2.0" +version = "0.2.1" dependencies = [ "krishiv-common", "krishiv-exec", diff --git a/Cargo.toml b/Cargo.toml index ae526fa57..0cb083f61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,12 +3,14 @@ resolver = "3" members = ["crates/*", "xtask"] [workspace.package] -version = "0.2.0" +version = "0.2.1" edition = "2024" # MSRV = latest stable (1.99) minus two minor releases. rust-version = "1.97" -license = "Apache-2.0" # ADR 0033 +license = "Apache-2.0" # ADR 0033 repository = "https://github.com/KrishivAI/krishiv-engine" +keywords = ["spark", "pyspark", "sql", "dataframe", "query-engine"] +categories = ["database-implementations", "database"] [workspace.lints.rust] unsafe_op_in_unsafe_fn = "deny" diff --git a/README.md b/README.md index 69cec538f..eff4c5e6c 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,21 @@ published TPC results. TPC-H is a trademark of the Transaction Processing Perfor - **Zero-copy interop** with pyarrow, Polars and other Arrow tools through the Arrow PyCapsule interface. - **`krishiv sql`**, a command-line SQL shell (scripts, table/CSV/JSON output, `--timing`). -The full list is in the [0.2.0 release notes](https://github.com/KrishivAI/krishiv-engine/releases/tag/v0.2.0). +The full list is in the [0.2.1 release notes](https://github.com/KrishivAI/krishiv-engine/releases/tag/v0.2.1). + +## Status and roadmap at a glance + +Krishiv is at **`0.2.1`**. The embedded core (in-process batch SQL and DataFrames over Arrow, from Python) runs; +streaming, incremental views, Spark Connect and distributed execution are still being built. + +| Milestone | Focus | Status | +|---|---|---| +| 1. Embedded core alpha | In-process SQL and DataFrames on Arrow, with Python and Rust APIs | ๐Ÿšง In progress | +| 2. Single-node (0.3) | `krishiv server` with Spark Connect, authentication, Iceberg and Delta, Python UDFs | โณ Planned | +| 3. Streaming | Stateful streaming with the same SQL and semantics as batch | โณ Planned | +| 4. Incremental views | Always-fresh materialized views and queryable state | โณ Planned | +| 5. Distributed | Scale out to a cluster with the same code | โณ Planned | +| 6. 1.0 | Stable APIs and formats | โณ Planned | ## Quickstart @@ -234,7 +248,7 @@ flowchart LR ```python from pyspark.sql import SparkSession, functions as F -# Planned for 0.3: a Krishiv server speaking the Spark Connect protocol. +# Planned: single-node Krishiv (`krishiv server`) speaking the Spark Connect protocol. spark = SparkSession.builder.remote("sc://localhost:15002").getOrCreate() orders = spark.read.parquet("s3://lake/orders/") diff --git a/RELEASING.md b/RELEASING.md index f5e5a1387..173ef4ea2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -133,6 +133,25 @@ Each step is a hard stop for agents; the maintainer runs it and approves its env 5. **Yank the old 0.1 releases** once 0.2.0 is verified: PyPI `krishiv` 0.1.0; crates.io `krishiv` 0.1.0-nightly.202607030148 and `krishiv-common` 0.1.0 (`cargo yank --version `). See below. 6. **GitHub Release** for `v0.2.0`, with the CHANGELOG 0.2.0 section as its notes. +## Publish tag (a `.github`-only fix after tagging) + +A release commit sometimes needs a fix under `.github/` only (a workflow, a script) after it is tagged. **Never move +or delete the release tag** (`v0.2.0`): a published tag is permanent. + +1. Make one commit that changes only files under `.github/`, directly on the release commit (not on `main`). +2. Tag it `vX.Y.Z-publishN` (for 0.2.0: `v0.2.0-publish`, `-publish2`, `-publish3`) and push the tag. +3. Dispatch the publish workflows (`publish-pypi.yml`, `publish-crates.yml`, and `publish-testpypi.yml` for a + rehearsal) on that tag. `.github/scripts/require-green-nightly.sh` accepts exactly one `.github`-only commit on a + parent that a green scheduled nightly covers; any second commit, or a commit that touches other paths, is refused. + +Two things learned from 0.2.0: + +- **crates.io rate-limits new crate names** (HTTP 429, "try again after "). `publish-crates.yml` waits until + the stated time and retries each crate (up to 40 attempts, a 330-minute job limit); a crate already published is + skipped, so a failed run can also be dispatched again. +- **0.2.0's bare `pip install krishiv` had no pyarrow**, and `collect()` needs it. From 0.2.1 pyarrow is a + required dependency; `krishiv[pyarrow]` stays as an alias. + ## Yanking a bad release (owner only) Every step is a hard stop for agents; the maintainer runs them. Yanking hides a version from new resolves and @@ -164,6 +183,19 @@ Owner decision 2026-11-07 (default adopted): a minor release every **6 weeks**. keeps its dates and the next minor takes what is ready. - Breaking changes (docs/concepts/stability.md) go into a minor only, with the changelog "Breaking" heading. +## Docs (every minor release) + +Owner decision 2026-11-14: the live docs of each minor version are read from a branch `docs/0.X`, not from a tag, so a +docs fix for a released version needs no new release. After the tag `v0.X.0` is pushed and the release is verified: + +1. **Cut the branch** `docs/0.X` from the tag `v0.X.0` and protect it like `main` (PRs only, docs CI runs on it). + Docs-only fixes for that version are PRs to `docs/0.X`; patch releases (`v0.X.Z`) are merged into it. +2. **Register the version** in krishiv-internal `site/docs-plugin.json`: add + `engine/0.X = Krishiv Engine 0.X | docs @ docs/0.X`, mark it latest, and leave `engine/next` on `main` (staging only). +3. **Deploy** with `site/deploy.sh`. +4. **Check** the version switcher lists the new version as latest and that the previous version still serves its own + pages. + ## Before any release - `ui/scripts/sync-assets.sh` refreshes the committed UI assets diff --git a/changelog.d/751.added.md b/changelog.d/751.added.md new file mode 100644 index 000000000..add3f534b --- /dev/null +++ b/changelog.d/751.added.md @@ -0,0 +1 @@ +(RS-15) The `krishiv` and `krishiv-cli` crates now have a `telemetry-otlp` feature that forwards to `krishiv-session/telemetry-otlp` (OTLP export of the query spans and engine metrics); 0.2.0 named it in the changelog but only `krishiv-session` had it. diff --git a/changelog.d/751.compat.md b/changelog.d/751.compat.md new file mode 100644 index 000000000..ec1ab9a7c --- /dev/null +++ b/changelog.d/751.compat.md @@ -0,0 +1 @@ +(SQL-1, PY-022, PY-006, RS-3) `pip install krishiv` now installs `pyarrow>=16`, which was optional before; the 0.2.1 wheels and crates carry PyPI and crates.io metadata and READMEs. diff --git a/changelog.d/751.fixed.md b/changelog.d/751.fixed.md new file mode 100644 index 000000000..0e8338fb6 --- /dev/null +++ b/changelog.d/751.fixed.md @@ -0,0 +1 @@ +(PY-021, PY-031, PY-032) The Python wheels now read and write `s3://`, `gs://` and `az://` URLs; the 0.2.0 wheels were built without cloud object stores and failed with `KRISHIV_UNSUPPORTED_FEATURE`. The new Data sources guide lists the credential, endpoint and region options. diff --git a/crates/krishiv-alloc/Cargo.toml b/crates/krishiv-alloc/Cargo.toml index cb2d487cb..923cbcc49 100644 --- a/crates/krishiv-alloc/Cargo.toml +++ b/crates/krishiv-alloc/Cargo.toml @@ -1,18 +1,21 @@ [package] name = "krishiv-alloc" description = "mimalloc settings and memory release for Krishiv binaries (never linked by libraries)" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] workspace = true [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } libmimalloc-sys = { workspace = true, features = ["extended"] } [features] diff --git a/crates/krishiv-alloc/README.md b/crates/krishiv-alloc/README.md new file mode 100644 index 000000000..4da974391 --- /dev/null +++ b/crates/krishiv-alloc/README.md @@ -0,0 +1,7 @@ +# krishiv-alloc + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because other Krishiv crates depend on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-catalogs/Cargo.toml b/crates/krishiv-catalogs/Cargo.toml index e5e1d593e..612c4b60d 100644 --- a/crates/krishiv-catalogs/Cargo.toml +++ b/crates/krishiv-catalogs/Cargo.toml @@ -16,9 +16,9 @@ workspace = true hive = ["dep:thrift", "dep:krishiv-plan", "dep:krishiv-types", "dep:tokio"] [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0", optional = true } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0", optional = true } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1", optional = true } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1", optional = true } tracing.workspace = true thrift = { workspace = true, optional = true } tokio = { workspace = true, features = ["rt"], optional = true } diff --git a/crates/krishiv-cli/Cargo.toml b/crates/krishiv-cli/Cargo.toml index 580d691e4..7e9e190e7 100644 --- a/crates/krishiv-cli/Cargo.toml +++ b/crates/krishiv-cli/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-cli" description = "Krishiv command line binary" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [[bin]] @@ -18,18 +21,18 @@ doc = false [dependencies] clap = { workspace = true, features = ["std", "derive", "help", "usage", "error-context"] } futures = { workspace = true, features = ["alloc", "executor"] } -krishiv = { path = "../krishiv", version = "=0.2.0", default-features = false } -krishiv-alloc = { path = "../krishiv-alloc", version = "=0.2.0", optional = true } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0", optional = true } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-sql = { path = "../krishiv-sql", version = "=0.2.0" } +krishiv = { path = "../krishiv", version = "=0.2.1", default-features = false } +krishiv-alloc = { path = "../krishiv-alloc", version = "=0.2.1", optional = true } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1", optional = true } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-sql = { path = "../krishiv-sql", version = "=0.2.1" } reedline = { workspace = true } -krishiv-server = { path = "../krishiv-server", version = "=0.2.0", optional = true } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0", optional = true } -krishiv-mcp = { path = "../krishiv-mcp", version = "=0.2.0", optional = true } -krishiv-udf-pool = { path = "../krishiv-udf-pool", version = "=0.2.0", optional = true } -krishiv-connect-server = { path = "../krishiv-connect-server", version = "=0.2.0", optional = true } +krishiv-server = { path = "../krishiv-server", version = "=0.2.1", optional = true } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1", optional = true } +krishiv-mcp = { path = "../krishiv-mcp", version = "=0.2.1", optional = true } +krishiv-connect-server = { path = "../krishiv-connect-server", version = "=0.2.1", optional = true } +krishiv-udf-pool = { path = "../krishiv-udf-pool", version = "=0.2.1", optional = true } axum = { workspace = true, optional = true } tokio = { workspace = true, features = ["rt", "rt-multi-thread"], optional = true } serde_json = { workspace = true } @@ -81,6 +84,8 @@ mimalloc = ["krishiv/mimalloc", "dep:mimalloc", "dep:krishiv-alloc", "dep:krishi tokio-console = ["krishiv/tokio-console"] # JSON logs on stderr with `KRISHIV_LOG=json` (T116). telemetry-logs = ["krishiv/telemetry-logs", "dep:krishiv-session"] +# OTLP export of the query spans and engine metrics. +telemetry-otlp = ["krishiv/telemetry-otlp", "dep:krishiv-session"] # Cloud object stores (S3, GCS, Azure). Opt-in so the default build stays # inside the dependency and binary-size budgets. cloud = ["krishiv/cloud"] diff --git a/crates/krishiv-cli/README.md b/crates/krishiv-cli/README.md new file mode 100644 index 000000000..59ff58521 --- /dev/null +++ b/crates/krishiv-cli/README.md @@ -0,0 +1,8 @@ +# krishiv-cli + +The `krishiv` command-line binary of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames +without the JVM. Install it with `cargo install krishiv-cli`, or use the prebuilt binaries from the +[releases page](https://github.com/KrishivAI/krishiv-engine/releases). To use the engine from Rust code, depend on the +[`krishiv`](https://crates.io/crates/krishiv) crate instead. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-cli/src/bundle.rs b/crates/krishiv-cli/src/bundle.rs index 508aff9a6..3f5361c10 100644 --- a/crates/krishiv-cli/src/bundle.rs +++ b/crates/krishiv-cli/src/bundle.rs @@ -518,63 +518,16 @@ pub(crate) fn feature_states() -> Vec<(&'static str, bool)> { ("celeborn", cfg!(feature = "celeborn")), ("tokio-console", cfg!(feature = "tokio-console")), ("telemetry-logs", cfg!(feature = "telemetry-logs")), + ("telemetry-otlp", cfg!(feature = "telemetry-otlp")), ("full", cfg!(feature = "full")), ] } fn enabled_features() -> Vec<&'static str> { - let mut out = Vec::new(); - if cfg!(feature = "mimalloc") { - out.push("mimalloc"); - } - if cfg!(feature = "server") { - out.push("server"); - } - if cfg!(feature = "ui") { - out.push("ui"); - } - if cfg!(feature = "kafka") { - out.push("kafka"); - } - if cfg!(feature = "iceberg") { - out.push("iceberg"); - } - if cfg!(feature = "delta") { - out.push("delta"); - } - if cfg!(feature = "cdc") { - out.push("cdc"); - } - if cfg!(feature = "connect") { - out.push("connect"); - } - if cfg!(feature = "flight-sql") { - out.push("flight-sql"); - } - if cfg!(feature = "mcp") { - out.push("mcp"); - } - if cfg!(feature = "python") { - out.push("python"); - } - if cfg!(feature = "collations-icu") { - out.push("collations-icu"); - } - if cfg!(feature = "wasm-udf") { - out.push("wasm-udf"); - } - if cfg!(feature = "celeborn") { - out.push("celeborn"); - } - if cfg!(feature = "tokio-console") { - out.push("tokio-console"); - } - if cfg!(feature = "telemetry-logs") { - out.push("telemetry-logs"); - } - if cfg!(feature = "full") { - out.push("full"); - } + let mut out: Vec<&'static str> = feature_states() + .into_iter() + .filter_map(|(name, on)| on.then_some(name)) + .collect(); out.sort_unstable(); out } diff --git a/crates/krishiv-common/Cargo.toml b/crates/krishiv-common/Cargo.toml index 0ac119ea1..a59332ab7 100644 --- a/crates/krishiv-common/Cargo.toml +++ b/crates/krishiv-common/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-common" description = "Errors, config registry and tracing setup" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] diff --git a/crates/krishiv-common/README.md b/crates/krishiv-common/README.md new file mode 100644 index 000000000..0894408cb --- /dev/null +++ b/crates/krishiv-common/README.md @@ -0,0 +1,7 @@ +# krishiv-common + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-connect-server/Cargo.toml b/crates/krishiv-connect-server/Cargo.toml index 74a71574c..b39717733 100644 --- a/crates/krishiv-connect-server/Cargo.toml +++ b/crates/krishiv-connect-server/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-connect-server" description = "Spark Connect gRPC server: the service, session manager and limits" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -17,12 +20,12 @@ arrow-ipc = { workspace = true, features = ["lz4", "zstd"] } arrow-schema.workspace = true futures = { workspace = true, features = ["std"] } bytes = { workspace = true, features = ["std"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-connect = { path = "../krishiv-connect", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0", features = ["unstable-ir"] } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-connect = { path = "../krishiv-connect", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1", features = ["unstable-ir"] } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } getrandom.workspace = true prost = { workspace = true, features = ["derive", "std"] } prost-types = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-connect-server/README.md b/crates/krishiv-connect-server/README.md new file mode 100644 index 000000000..cc51cb387 --- /dev/null +++ b/crates/krishiv-connect-server/README.md @@ -0,0 +1,7 @@ +# krishiv-connect-server + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because other Krishiv crates depend on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-connect/Cargo.toml b/crates/krishiv-connect/Cargo.toml index 003fa86ed..2647bba73 100644 --- a/crates/krishiv-connect/Cargo.toml +++ b/crates/krishiv-connect/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-connect" description = "Spark Connect server" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -14,9 +17,9 @@ workspace = true [dependencies] arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } # `no-recursion-limit`: prost's fixed limit of 100 nested messages rejects # ordinary plans (a loop of `withColumn` calls); `wire::decode_plan` bounds the # depth itself, before prost recurses (see the crate CLAUDE.md). diff --git a/crates/krishiv-connect/README.md b/crates/krishiv-connect/README.md new file mode 100644 index 000000000..2ac993cfd --- /dev/null +++ b/crates/krishiv-connect/README.md @@ -0,0 +1,7 @@ +# krishiv-connect + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-connectors-delta/Cargo.toml b/crates/krishiv-connectors-delta/Cargo.toml index 6b49fbb33..ae37fbce9 100644 --- a/crates/krishiv-connectors-delta/Cargo.toml +++ b/crates/krishiv-connectors-delta/Cargo.toml @@ -26,12 +26,12 @@ arrow-select.workspace = true delta_kernel = { workspace = true, features = ["internal-api"] } bytes.workspace = true chrono.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } -krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } +krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } object_store.workspace = true serde_json = { workspace = true, features = ["std"] } tokio = { workspace = true, features = ["macros", "rt", "sync"] } diff --git a/crates/krishiv-exec/Cargo.toml b/crates/krishiv-exec/Cargo.toml index cc04c4648..aa3525f48 100644 --- a/crates/krishiv-exec/Cargo.toml +++ b/crates/krishiv-exec/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-exec" description = "Batch execution engine" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -21,15 +24,15 @@ arrow-select.workspace = true foldhash.workspace = true hashbrown.workspace = true futures = { workspace = true, features = ["alloc"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } -krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } +krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } thiserror.workspace = true # The query profile's JSON document (`profile`). serde_json = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-exec/README.md b/crates/krishiv-exec/README.md new file mode 100644 index 000000000..35189ba58 --- /dev/null +++ b/crates/krishiv-exec/README.md @@ -0,0 +1,7 @@ +# krishiv-exec + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-functions/Cargo.toml b/crates/krishiv-functions/Cargo.toml index 6f48a1cdb..6eca37ada 100644 --- a/crates/krishiv-functions/Cargo.toml +++ b/crates/krishiv-functions/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-functions" description = "Spark-compatible function library" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] @@ -26,10 +29,10 @@ regex-automata = { workspace = true, features = [ "unicode-word-boundary", ] } regex-syntax = { workspace = true, features = ["unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } quick-xml.workspace = true aes.workspace = true cbc = { workspace = true, features = ["alloc", "block-padding"] } diff --git a/crates/krishiv-functions/README.md b/crates/krishiv-functions/README.md new file mode 100644 index 000000000..53630a09a --- /dev/null +++ b/crates/krishiv-functions/README.md @@ -0,0 +1,7 @@ +# krishiv-functions + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-io/Cargo.toml b/crates/krishiv-io/Cargo.toml index c68ef1307..d1342b405 100644 --- a/crates/krishiv-io/Cargo.toml +++ b/crates/krishiv-io/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-io" description = "Object store and file format I/O" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -30,13 +33,13 @@ serde_json = { workspace = true, features = ["std"] } # Exact runtime-filter sets, placed by the row hash (P2-T163). hashbrown.workspace = true futures = { workspace = true, features = ["alloc", "std"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } # Spark value text conversions and datetime patterns (CSV, JSON, text). -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } # URL parsing, store building and pacing budgets (ADR 0044); `cloud` turns on its builders. -krishiv-store = { path = "../krishiv-store", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-store = { path = "../krishiv-store", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } memchr.workspace = true # `fs`: the local store reads use wraps `LocalFileSystem` (`io/local_listing.rs`). object_store = { workspace = true, features = ["fs"] } diff --git a/crates/krishiv-io/README.md b/crates/krishiv-io/README.md new file mode 100644 index 000000000..fe7ea4bdb --- /dev/null +++ b/crates/krishiv-io/README.md @@ -0,0 +1,7 @@ +# krishiv-io + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-kernels/Cargo.toml b/crates/krishiv-kernels/Cargo.toml index 06dfe8477..48e652902 100644 --- a/crates/krishiv-kernels/Cargo.toml +++ b/crates/krishiv-kernels/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-kernels" description = "Compute kernels (SIMD)" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -18,8 +21,8 @@ arrow-schema.workspace = true arrow-select.workspace = true arrow-string.workspace = true arrow-ord.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } jiff = { workspace = true, features = ["std", "tzdb-bundle-always"] } libm.workspace = true memchr = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-kernels/README.md b/crates/krishiv-kernels/README.md new file mode 100644 index 000000000..8c24a8b5f --- /dev/null +++ b/crates/krishiv-kernels/README.md @@ -0,0 +1,7 @@ +# krishiv-kernels + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-mcp/Cargo.toml b/crates/krishiv-mcp/Cargo.toml index 35c8df4a9..b5f13f26b 100644 --- a/crates/krishiv-mcp/Cargo.toml +++ b/crates/krishiv-mcp/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-mcp" description = "MCP adapter" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -14,10 +17,10 @@ workspace = true [dependencies] rmcp = { workspace = true, features = ["server", "transport-io"] } futures.workspace = true -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0" } -krishiv-sql = { path = "../krishiv-sql", version = "=0.2.0" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1" } +krishiv-sql = { path = "../krishiv-sql", version = "=0.2.1" } serde_json = { workspace = true, features = ["std"] } # The `mcp_stdio` conformance shim serves stdio from a plain binary (not a # test harness), so its stdout carries only JSON-RPC. diff --git a/crates/krishiv-mcp/README.md b/crates/krishiv-mcp/README.md new file mode 100644 index 000000000..10ccc780a --- /dev/null +++ b/crates/krishiv-mcp/README.md @@ -0,0 +1,7 @@ +# krishiv-mcp + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because other Krishiv crates depend on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-memory/Cargo.toml b/crates/krishiv-memory/Cargo.toml index b28280f4a..20b6ad07e 100644 --- a/crates/krishiv-memory/Cargo.toml +++ b/crates/krishiv-memory/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-memory" description = "Memory management" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -17,8 +20,8 @@ arrow-buffer.workspace = true arrow-ipc = { workspace = true, features = ["lz4", "zstd"] } arrow-schema.workspace = true sha2 = { workspace = true } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } thiserror.workspace = true tracing = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-memory/README.md b/crates/krishiv-memory/README.md new file mode 100644 index 000000000..982eb6816 --- /dev/null +++ b/crates/krishiv-memory/README.md @@ -0,0 +1,7 @@ +# krishiv-memory + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-optimizer/Cargo.toml b/crates/krishiv-optimizer/Cargo.toml index 05de1850d..31698faf8 100644 --- a/crates/krishiv-optimizer/Cargo.toml +++ b/crates/krishiv-optimizer/Cargo.toml @@ -1,22 +1,25 @@ [package] name = "krishiv-optimizer" description = "Query optimizer" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } [dev-dependencies] criterion.workspace = true diff --git a/crates/krishiv-optimizer/README.md b/crates/krishiv-optimizer/README.md new file mode 100644 index 000000000..8fd3bfb0f --- /dev/null +++ b/crates/krishiv-optimizer/README.md @@ -0,0 +1,7 @@ +# krishiv-optimizer + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-plan-codec/Cargo.toml b/crates/krishiv-plan-codec/Cargo.toml index 29edefc78..617233674 100644 --- a/crates/krishiv-plan-codec/Cargo.toml +++ b/crates/krishiv-plan-codec/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-plan-codec" description = "The krishiv.plan.v1 codec for physical plans" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -14,11 +17,11 @@ workspace = true [dependencies] arrow-ipc.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } prost = { workspace = true, features = ["derive", "std"] } thiserror.workspace = true diff --git a/crates/krishiv-plan-codec/README.md b/crates/krishiv-plan-codec/README.md new file mode 100644 index 000000000..c0a31e577 --- /dev/null +++ b/crates/krishiv-plan-codec/README.md @@ -0,0 +1,7 @@ +# krishiv-plan-codec + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-plan/Cargo.toml b/crates/krishiv-plan/Cargo.toml index 6b74a7ee3..73b67f70a 100644 --- a/crates/krishiv-plan/Cargo.toml +++ b/crates/krishiv-plan/Cargo.toml @@ -1,27 +1,30 @@ [package] name = "krishiv-plan" description = "Logical and physical plans" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] # One-row arrays: constant arguments fold through the cast kernels. arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } # The built-in function registry and the signature types it shares. -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } # The CSV literal inference `schema_of_csv` resolves with (layer L2). -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } # Spark's decimal result types, shared with the kernels that compute them. -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } # Session UDFs, for constant folding through the registry. -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } thiserror.workspace = true twox-hash = { workspace = true, features = ["xxhash3_128"] } diff --git a/crates/krishiv-plan/README.md b/crates/krishiv-plan/README.md new file mode 100644 index 000000000..cdba623cd --- /dev/null +++ b/crates/krishiv-plan/README.md @@ -0,0 +1,7 @@ +# krishiv-plan + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-python/Cargo.toml b/crates/krishiv-python/Cargo.toml index 780a7b0d6..92b6f11c5 100644 --- a/crates/krishiv-python/Cargo.toml +++ b/crates/krishiv-python/Cargo.toml @@ -28,11 +28,14 @@ doctest = false [features] # Default wheel: one abi3 wheel for CPython >= 3.11. The free-threaded (cp314t) # build cannot use abi3, so it builds with -# `--no-default-features --features mimalloc`. -default = ["abi3-py311", "mimalloc"] +# `--no-default-features --features mimalloc,cloud`. +default = ["abi3-py311", "mimalloc", "cloud"] abi3-py311 = ["pyo3/abi3-py311"] # mimalloc is the global allocator of the extension module (P2-T012). mimalloc = ["dep:mimalloc"] +# Cloud object stores (s3://, gs://, az://) in every wheel: without it they fail with +# KRISHIV_UNSUPPORTED_FEATURE. 0.2.0 shipped without it. +cloud = ["krishiv-session/cloud"] [dependencies] arrow-array = { workspace = true, features = ["ffi"] } diff --git a/crates/krishiv-server/Cargo.toml b/crates/krishiv-server/Cargo.toml index 654021800..9b1154030 100644 --- a/crates/krishiv-server/Cargo.toml +++ b/crates/krishiv-server/Cargo.toml @@ -1,21 +1,24 @@ [package] name = "krishiv-server" description = "Admin and UI API server" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] axum = { workspace = true, features = ["http1", "tokio"] } arrow-array = { workspace = true, optional = true } arrow-schema = { workspace = true, optional = true } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0", optional = true } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0", optional = true } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0", optional = true } -krishiv-ui-assets = { path = "../krishiv-ui-assets", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1", optional = true } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1", optional = true } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1", optional = true } +krishiv-ui-assets = { path = "../krishiv-ui-assets", version = "=0.2.1" } futures = { workspace = true, features = ["alloc"] } getrandom.workspace = true serde_json = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-server/README.md b/crates/krishiv-server/README.md new file mode 100644 index 000000000..4b43b275e --- /dev/null +++ b/crates/krishiv-server/README.md @@ -0,0 +1,7 @@ +# krishiv-server + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-session/Cargo.toml b/crates/krishiv-session/Cargo.toml index 4cb007c28..487978a16 100644 --- a/crates/krishiv-session/Cargo.toml +++ b/crates/krishiv-session/Cargo.toml @@ -15,18 +15,18 @@ arrow-schema.workspace = true thiserror = { workspace = true, optional = true } # `executor`: file footers are read on a thread of their own (catalog.rs). futures = { workspace = true, features = ["alloc", "executor"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0" } -krishiv-functions = { path = "../krishiv-functions", version = "=0.2.0" } -krishiv-io = { path = "../krishiv-io", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-memory = { path = "../krishiv-memory", version = "=0.2.0" } -krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-plan-codec = { path = "../krishiv-plan-codec", version = "=0.2.0" } -krishiv-sql = { path = "../krishiv-sql", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } -krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1" } +krishiv-functions = { path = "../krishiv-functions", version = "=0.2.1" } +krishiv-io = { path = "../krishiv-io", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-memory = { path = "../krishiv-memory", version = "=0.2.1" } +krishiv-optimizer = { path = "../krishiv-optimizer", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-plan-codec = { path = "../krishiv-plan-codec", version = "=0.2.1" } +krishiv-sql = { path = "../krishiv-sql", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } +krishiv-udf-sdk = { path = "../krishiv-udf-sdk", version = "=0.2.1" } # The `EXPLAIN VERBOSE` JSON document; OIDC token and JWK set parsing (`auth/jose.rs`). serde = { workspace = true, features = ["std"] } serde_json = { workspace = true, features = ["std"] } @@ -61,7 +61,7 @@ object_store = { workspace = true, features = ["fs"] } [dev-dependencies] bytes.workspace = true -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0", features = ["testing"] } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1", features = ["testing"] } arrow-select.workspace = true futures = { workspace = true, features = ["alloc", "executor"] } insta.workspace = true diff --git a/crates/krishiv-session/README.md b/crates/krishiv-session/README.md new file mode 100644 index 000000000..2be836949 --- /dev/null +++ b/crates/krishiv-session/README.md @@ -0,0 +1,7 @@ +# krishiv-session + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-sql/Cargo.toml b/crates/krishiv-sql/Cargo.toml index 5893b7c29..b13edfe6d 100644 --- a/crates/krishiv-sql/Cargo.toml +++ b/crates/krishiv-sql/Cargo.toml @@ -1,21 +1,24 @@ [package] name = "krishiv-sql" description = "SQL parsing and analysis" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] workspace = true [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.0" } -krishiv-plan = { path = "../krishiv-plan", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-kernels = { path = "../krishiv-kernels", version = "=0.2.1" } +krishiv-plan = { path = "../krishiv-plan", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } thiserror.workspace = true [dev-dependencies] diff --git a/crates/krishiv-sql/README.md b/crates/krishiv-sql/README.md new file mode 100644 index 000000000..734bf3e7c --- /dev/null +++ b/crates/krishiv-sql/README.md @@ -0,0 +1,7 @@ +# krishiv-sql + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-store/Cargo.toml b/crates/krishiv-store/Cargo.toml index 837973789..8bca672e7 100644 --- a/crates/krishiv-store/Cargo.toml +++ b/crates/krishiv-store/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-store" description = "Object-store layer shared by the Krishiv engine and platform: URLs, store building, credentials (ADR 0044)" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true # Meant to be published (ADR 0044); stays unpublished until the owner's first release of it. publish = true @@ -14,7 +17,7 @@ workspace = true # krishiv-common and object_store only (layer L1, ADR 0044). [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } # Bearer header of `hf://` reads; only with `cloud`. http = { workspace = true, optional = true } object_store.workspace = true diff --git a/crates/krishiv-store/README.md b/crates/krishiv-store/README.md new file mode 100644 index 000000000..a20538941 --- /dev/null +++ b/crates/krishiv-store/README.md @@ -0,0 +1,7 @@ +# krishiv-store + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-types/Cargo.toml b/crates/krishiv-types/Cargo.toml index f8a9e9e8f..863c5c25e 100644 --- a/crates/krishiv-types/Cargo.toml +++ b/crates/krishiv-types/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-types" description = "Internal type representations" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [lints] @@ -16,7 +19,7 @@ arrow-array.workspace = true arrow-buffer.workspace = true arrow-schema.workspace = true arrow-select.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } serde_json = { workspace = true, features = ["std"] } [dev-dependencies] diff --git a/crates/krishiv-types/README.md b/crates/krishiv-types/README.md new file mode 100644 index 000000000..557f1c397 --- /dev/null +++ b/crates/krishiv-types/README.md @@ -0,0 +1,7 @@ +# krishiv-types + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-udf-pool/Cargo.toml b/crates/krishiv-udf-pool/Cargo.toml index 4084eb7fb..07cb1fc46 100644 --- a/crates/krishiv-udf-pool/Cargo.toml +++ b/crates/krishiv-udf-pool/Cargo.toml @@ -16,9 +16,9 @@ arrow-array.workspace = true arrow-ipc.workspace = true arrow-schema.workspace = true futures = { workspace = true, features = ["std"] } -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0" } -krishiv-udf-sandbox = { path = "../krishiv-udf-sandbox", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1" } +krishiv-udf-sandbox = { path = "../krishiv-udf-sandbox", version = "=0.2.1" } serde_json = { workspace = true, features = ["std"] } tempfile.workspace = true tracing = { workspace = true, features = ["std"] } diff --git a/crates/krishiv-udf-sandbox/Cargo.toml b/crates/krishiv-udf-sandbox/Cargo.toml index 99bb81b3e..8ea93c290 100644 --- a/crates/krishiv-udf-sandbox/Cargo.toml +++ b/crates/krishiv-udf-sandbox/Cargo.toml @@ -12,7 +12,7 @@ publish = true workspace = true [dependencies] -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } thiserror.workspace = true [target.'cfg(target_os = "linux")'.dependencies] diff --git a/crates/krishiv-udf-sdk/Cargo.toml b/crates/krishiv-udf-sdk/Cargo.toml index 465593f08..06c1ff91d 100644 --- a/crates/krishiv-udf-sdk/Cargo.toml +++ b/crates/krishiv-udf-sdk/Cargo.toml @@ -1,16 +1,19 @@ [package] name = "krishiv-udf-sdk" description = "UDF SDK" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] arrow-array.workspace = true -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } [lints] workspace = true diff --git a/crates/krishiv-udf-sdk/README.md b/crates/krishiv-udf-sdk/README.md new file mode 100644 index 000000000..c4d26b113 --- /dev/null +++ b/crates/krishiv-udf-sdk/README.md @@ -0,0 +1,7 @@ +# krishiv-udf-sdk + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv-ui-assets/Cargo.toml b/crates/krishiv-ui-assets/Cargo.toml index 4265befb0..e0e352ffc 100644 --- a/crates/krishiv-ui-assets/Cargo.toml +++ b/crates/krishiv-ui-assets/Cargo.toml @@ -1,11 +1,14 @@ [package] name = "krishiv-ui-assets" description = "Embedded web UI assets" +readme = "README.md" version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true +keywords.workspace = true +categories.workspace = true publish = true [dependencies] diff --git a/crates/krishiv-ui-assets/README.md b/crates/krishiv-ui-assets/README.md new file mode 100644 index 000000000..ef7f8136e --- /dev/null +++ b/crates/krishiv-ui-assets/README.md @@ -0,0 +1,7 @@ +# krishiv-ui-assets + +An internal crate of [Krishiv](https://krishiv.ai), a Spark alternative in Rust: Spark SQL and DataFrames without the JVM. +It is published only because the `krishiv` crate depends on it. Use the [`krishiv`](https://crates.io/crates/krishiv) +crate instead; this crate's API is not stable and can change in any release. + +[Website](https://krishiv.ai) ยท [Documentation](https://krishiv.ai/docs) ยท [Source](https://github.com/KrishivAI/krishiv-engine) ยท Apache-2.0 diff --git a/crates/krishiv/Cargo.toml b/crates/krishiv/Cargo.toml index e4dbaa46f..fa1e7f9e9 100644 --- a/crates/krishiv/Cargo.toml +++ b/crates/krishiv/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "krishiv" -description = "Spark-compatible SQL and DataFrame engine in Rust: embedded, vectorized on Arrow, no JVM" -keywords = ["spark", "sql", "dataframe", "arrow", "query-engine"] -categories = ["database-implementations", "data-structures"] +description = "Spark alternative in Rust: run Spark SQL and DataFrames embedded, vectorized on Arrow, no JVM" +readme = "README.md" +keywords = ["spark", "pyspark", "sql", "dataframe", "query-engine"] +categories = ["database-implementations", "database"] homepage = "https://krishiv.ai" documentation = "https://docs.rs/krishiv" version.workspace = true @@ -18,11 +19,11 @@ workspace = true [dependencies] arrow-array.workspace = true arrow-schema.workspace = true -krishiv-common = { path = "../krishiv-common", version = "=0.2.0" } -krishiv-exec = { path = "../krishiv-exec", version = "=0.2.0", optional = true } -krishiv-server = { path = "../krishiv-server", version = "=0.2.0", optional = true } -krishiv-session = { path = "../krishiv-session", version = "=0.2.0" } -krishiv-types = { path = "../krishiv-types", version = "=0.2.0" } +krishiv-common = { path = "../krishiv-common", version = "=0.2.1" } +krishiv-exec = { path = "../krishiv-exec", version = "=0.2.1", optional = true } +krishiv-server = { path = "../krishiv-server", version = "=0.2.1", optional = true } +krishiv-session = { path = "../krishiv-session", version = "=0.2.1" } +krishiv-types = { path = "../krishiv-types", version = "=0.2.1" } [dev-dependencies] futures = { workspace = true, features = ["alloc", "executor"] } @@ -56,4 +57,6 @@ tokio-console = ["dep:krishiv-exec", "krishiv-exec/tokio-console"] # JSON log records with query, session and trace ids (T116), installed by the CLI # when `KRISHIV_LOG=json`; embedders call `krishiv_session::logging` themselves. telemetry-logs = ["krishiv-session/telemetry-logs"] +# OTLP export of the query spans and engine metrics (`krishiv_session::otlp`), off by default. +telemetry-otlp = ["krishiv-session/telemetry-otlp"] full = ["kafka", "iceberg", "delta", "cdc", "connect", "flight-sql", "mcp", "python", "wasm-udf", "ui", "celeborn", "mimalloc", "cloud"] diff --git a/crates/krishiv/README.md b/crates/krishiv/README.md new file mode 100644 index 000000000..68b29b6fe --- /dev/null +++ b/crates/krishiv/README.md @@ -0,0 +1,52 @@ +# krishiv + +**Spark alternative in Rust: run Spark SQL and DataFrames embedded, vectorized on Arrow, no JVM.** + +Krishiv runs inside your process: no cluster and no JVM. Queries follow Spark's SQL and DataFrame semantics and +error classes, execute vectorized on Apache Arrow, and return Arrow record batches. + +```bash +cargo add krishiv +``` + +```rust +use std::sync::Arc; +use krishiv::arrow::array::{Int32Array, RecordBatch}; +use krishiv::arrow::datatypes::{DataType, Field, Schema}; +use krishiv::functions::col; +use krishiv::Session; + +let session = Session::new()?; +let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); +let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![3, 1, 2]))]).unwrap(); +session.register_batches("t", schema, vec![batch])?; + +// DataFrames and SQL run on the same engine. +let top = session.table("t").order_by([col("a").desc()]).limit(2); +let rows: Vec = futures::executor::block_on(top.collect())?; +assert_eq!(rows.iter().map(|b| b.num_rows()).sum::(), 2); +# Ok::<(), krishiv::Error>(()) +``` + +Actions are `async` and run on the session's own executor, so any runtime (or none, with +`futures::executor::block_on`) can await them. + +## Speed + +Benchmark results against Spark 4.2.0 on TPC-H-derived queries (not official TPC results) are published with their +method and numbers at . + +## Status + +Alpha. The embedded batch core works; streaming, incremental views, Spark Connect and distributed execution are +still being built. There is no stable API yet. Python users: `pip install krishiv`. + +## Links + +- Website: +- Documentation: +- Source and issues: +- Releases and changelog: + +Licensed under Apache-2.0. Apache Spark is a trademark of the Apache Software Foundation; Krishiv is not +affiliated with it. diff --git a/crates/krishiv/src/lib.rs b/crates/krishiv/src/lib.rs index 333f7802d..5812b5068 100644 --- a/crates/krishiv/src/lib.rs +++ b/crates/krishiv/src/lib.rs @@ -29,6 +29,11 @@ #![forbid(unsafe_code)] +// The crate README is a doc test: its example is compiled and run by `cargo test --doc`. +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeDoctests; + pub use krishiv_common::error::KrishivError as Error; pub use krishiv_session::{ AggregateProperties, AggregateUdf, ArgField, CancelReason, Column, CounterSample, DataFrame, diff --git a/docs/guides/_meta.json b/docs/guides/_meta.json index a3f86e01c..03592f579 100644 --- a/docs/guides/_meta.json +++ b/docs/guides/_meta.json @@ -5,5 +5,6 @@ "check-compat": "Check a project with check-compat", "run-sql-from-the-command-line": "Run SQL from the command line", "tokio-console": "Task diagnostics with tokio-console", + "data-sources": "Data sources: files, S3, GCS and Azure", "notebooks-and-datasets": "Notebooks and sample data" } diff --git a/docs/guides/data-sources.md b/docs/guides/data-sources.md new file mode 100644 index 000000000..9fdc1d7e1 --- /dev/null +++ b/docs/guides/data-sources.md @@ -0,0 +1,79 @@ +--- +description: "Read and write Parquet, CSV and other files on local disk and on S3, Google Cloud Storage and Azure Blob Storage." +--- + +# Data sources + +Krishiv reads and writes files from your own disk and, from 0.2.1, from cloud object stores: Amazon S3 (`s3://`, `s3a://`), +Google Cloud Storage (`gs://`) and Azure Blob Storage (`az://`, `abfs://`, `abfss://`). The wheels from PyPI include the +object-store support; the 0.2.0 wheels did not, and failed with `KRISHIV_UNSUPPORTED_FEATURE` on these URLs. + +```python +import krishiv + +spark = krishiv.session() +spark.conf.set("fs.s3a.endpoint.region", "eu-west-1") +df = spark.read.parquet("s3://my-bucket/events/") +df.write.mode("overwrite").parquet("s3://my-bucket/summary/") +``` + +## Credentials and options + +Set the options on the session with `spark.conf.set`, or for one read or write with `.option(key, value)`. A per-read +option overrides the session value, and two reads of one bucket with different credentials never share a connection. +Keys match without regard to case. An option value is never printed: errors and `EXPLAIN` name the key only. + +When an option is not set, Krishiv falls back to the standard environment variables of each cloud (`AWS_ACCESS_KEY_ID`, +`AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, `GOOGLE_APPLICATION_CREDENTIALS`, `AZURE_STORAGE_ACCOUNT_NAME`, and so on). + +### Amazon S3 and S3-compatible stores + +| Option | Meaning | +|---|---| +| `fs.s3a.access.key`, `fs.s3a.secret.key` | access key id and secret | +| `fs.s3a.session.token` | session token, for temporary credentials | +| `fs.s3a.endpoint.region` (or `fs.s3a.region`) | region; the default is `us-east-1` | +| `fs.s3a.endpoint` | endpoint URL for MinIO, LocalStack, Ceph and similar; `http://` is allowed | +| `fs.s3a.path.style.access` | `true` forces `https://host/bucket/key` requests; already the default for `http://` and `localhost` endpoints | + +### Google Cloud Storage + +| Option | Meaning | +|---|---| +| `fs.gs.auth.service.account.json.keyfile` | path to a service-account JSON file | +| `fs.gs.auth.service.account.json` | the service-account JSON itself | +| `fs.gs.storage.root.url` (or `fs.gs.endpoint`) | endpoint URL for an emulator | + +### Azure Blob Storage + +| Option | Meaning | +|---|---| +| `fs.azure.account.key..dfs.core.windows.net` | storage account key (the `.blob.` form works too) | +| `fs.azure.sas...dfs.core.windows.net` | shared access signature for one container | +| `fs.azure.account.name` | account name, when the URL is `az://container/path` and does not carry it | +| `fs.azure.endpoint` | endpoint URL for Azurite (`http://127.0.0.1:10000/`) | + +## Try it without a cloud account + +Run MinIO locally, create a bucket as a directory of its data folder, and point `fs.s3a.endpoint` at it: + +```bash +mkdir -p data/my-bucket +docker run --rm -p 9000:9000 -v "$PWD/data:/data" quay.io/minio/minio server /data +``` + +```python +for key, value in { + "fs.s3a.endpoint": "http://127.0.0.1:9000", + "fs.s3a.access.key": "minioadmin", + "fs.s3a.secret.key": "minioadmin", +}.items(): + spark.conf.set(key, value) +spark.read.parquet("s3://my-bucket/events/").show() +``` + +## On a shared server + +In server mode with authentication on, reads and writes by path are confined to the prefixes in +`krishiv.io.objectStore.allowedPrefixes` and `krishiv.io.local.allowedRoots`; see the +[configuration reference](../reference/config/index.md). diff --git a/docs/index.md b/docs/index.md index 53d1ee0fa..aaabfe7ad 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,13 +8,11 @@ Krishiv is a Rust-native, Apache Spark-compatible compute engine: batch SQL, Dat maintained views on one vectorized, Arrow-native engine, with no JVM. Keep the PySpark and Spark SQL you already write. -> **Alpha (`0.2.0-alpha`).** The embedded core runs: in-process batch SQL and DataFrames from Python. There is no -> stable API yet. Streaming, incremental views, Spark Connect and distributed execution are being built. +> Krishiv 0.2 is released on PyPI and crates.io. APIs can change before 1.0. ## Install -Nothing is on PyPI yet, so build the wheel from the repository (`maturin build` in `python/`) and install it. Once -released, `pip install "krishiv[pyarrow]"` is all it takes; the [Quickstart](get-started/quickstart.md) has the steps. +Install with `pip install "krishiv[pyarrow]"`; the [Quickstart](get-started/quickstart.md) has the steps. ## Where to start diff --git a/docs/reference/python-package.md b/docs/reference/python-package.md index 4e0095a4c..d5c83a720 100644 --- a/docs/reference/python-package.md +++ b/docs/reference/python-package.md @@ -20,7 +20,7 @@ table = df.toArrow() # a pyarrow.Table - Actions (`collect`, `show`, `toArrow`, `toPandas`, `count`) run the query in-process with the GIL released. - A DataFrame implements the Arrow PyCapsule interface, so `pa.table(df)`, `polars.DataFrame(df)` and `duckdb.from_arrow(df)` read the result without copying. -- pyarrow is optional (`pip install "krishiv[pyarrow]"`); `toPandas` also needs pandas (`krishiv[pandas]`). +- pyarrow is a required dependency (since 0.2.1; `krishiv[pyarrow]` still works as an alias); `toPandas` also needs pandas (`krishiv[pandas]`). - `df.profile()` (or `krishiv.ui(df)`) returns a viewer of the query's profile; in a notebook it renders itself. - Test your PySpark code without a JVM: `krishiv.testing` has `assertDataFrameEqual`, `assertSchemaEqual` and a pytest fixture (`pytest_plugins = ["krishiv.testing.pytest_plugin"]`, then use the `krishiv_session` fixture). diff --git a/python/PYPI.md b/python/PYPI.md new file mode 100644 index 000000000..03247e03a --- /dev/null +++ b/python/PYPI.md @@ -0,0 +1,65 @@ +# Krishiv + +**Spark alternative in Rust: PySpark and Spark SQL without the JVM.** + +Krishiv runs Spark SQL and a PySpark-style DataFrame API inside your Python process. No cluster, no JVM, no +server to start. Execution is vectorized on Apache Arrow, and results come back as Arrow tables you can hand to +pandas, Polars or DuckDB without copying. + +## Install + +```bash +pip install krishiv +``` + +Needs Python 3.11 or newer. `pyarrow` is installed with it. For `toPandas()` add the extra: +`pip install "krishiv[pandas]"`. + +## 30-second example + +```python +import krishiv +from krishiv.sql import functions as F + +spark = krishiv.session() # a PySpark-style SparkSession, in process + +spark.sql("SELECT 1 AS answer").show() + +df = spark.sql("SELECT * FROM VALUES ('east', 10.0), ('west', 20.0), ('east', 5.5) AS t(region, amount)") +df.groupBy("region").agg(F.sum("amount").alias("revenue")).orderBy("region").show() + +table = df.toArrow() # a pyarrow.Table; also pa.table(df), polars.DataFrame(df), duckdb.from_arrow(df) +``` + +## What works in 0.2 + +- Batch Spark SQL and DataFrames, in process, over Arrow. `krishiv.sql` mirrors `pyspark.sql` + (import paths and signatures); errors in `krishiv.errors` carry Spark's error classes. +- Reading and writing Parquet files; `EXPLAIN` and `EXPLAIN ANALYZE` with per-operator rows, time and memory. +- Spark's function library, with Spark's results (hashing, casts, null handling and error behaviour follow Spark). +- `krishiv.testing`: `assertDataFrameEqual` and a pytest fixture, to run your PySpark tests without a JVM. + +Streaming, incremental views, Spark Connect and distributed execution are still being built. This is an alpha: +there is no stable API yet. + +## Speed + +Benchmark results against Spark 4.2.0 on TPC-H-derived queries (not official TPC results) are published with their +method, queries and raw numbers at . + +## Supported CPUs + +The default wheels need an x86-64-v3 CPU (Intel Haswell, 2013, or AMD Zen and newer) or, on arm64, Armv8.2 (AWS +Graviton2, Apple silicon). On an older x86-64 CPU install `krishiv-compat` instead: the same package and the +same `import krishiv`, built for the plain x86-64 baseline (Linux only). Install one of the two, not both. + +## Links + +- Website: +- Documentation: +- Benchmarks: +- Source and issues: +- Releases and changelog: + +Krishiv is licensed under Apache-2.0. Apache Spark is a trademark of the Apache Software Foundation; Krishiv is +not affiliated with it. diff --git a/python/README.md b/python/README.md index 5afacb89e..1aa092eed 100644 --- a/python/README.md +++ b/python/README.md @@ -29,8 +29,7 @@ DataFrame also implements the Arrow PyCapsule interface (`__arrow_c_stream__`), and releasing the stream cancels it. `krishiv.sql` mirrors `pyspark.sql` (signatures and import paths); errors are in `krishiv.errors`, shaped like -`pyspark.errors`. pyarrow is optional: install `krishiv[pyarrow]`; `toArrow`, `collect`, `show` and `toPandas` need it (`count` and -the PyCapsule interface do not). `toPandas` also needs pandas (`pip install krishiv[pandas]`). SQL expression strings +`pyspark.errors`. pyarrow is a required dependency since 0.2.1 (`krishiv[pyarrow]` still works as an alias); `toArrow`, `collect`, `show` and `toPandas` use it. `toPandas` also needs pandas (`pip install krishiv[pandas]`). SQL expression strings are supported (`F.expr`, `selectExpr`, `filter`/`where` with a string). ## Run your PySpark tests locally @@ -106,7 +105,7 @@ non-abi3 build (disable the default `abi3-py311` crate feature but keep `mimallo cd python uv venv --python 3.14t .venv-314t && uv pip install --python .venv-314t/bin/python maturin pytest . .venv-314t/bin/activate -maturin develop --no-default-features --features mimalloc # or: maturin build ... +maturin develop --no-default-features --features mimalloc,cloud # or: maturin build ... # (mimalloc is a default feature; keep it when turning the defaults off) ``` diff --git a/python/pyproject.toml b/python/pyproject.toml index a9baf99c6..ed22b1ee4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,11 +4,18 @@ build-backend = "maturin" [project] name = "krishiv" -description = "Krishiv: a Rust-native, Spark-compatible batch and streaming engine" +description = "Spark alternative in Rust: PySpark and Spark SQL without the JVM" +readme = "PYPI.md" requires-python = ">=3.11" license = { text = "Apache-2.0" } +keywords = ["spark", "pyspark", "spark-alternative", "sql", "dataframe", "arrow", "rust", "etl", "big-data"] +dependencies = ["pyarrow>=16"] dynamic = ["version"] classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: Apache Software License", + "Topic :: Database", + "Topic :: Scientific/Engineering", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.11", @@ -20,11 +27,17 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", ] +[project.urls] +Homepage = "https://krishiv.ai" +Documentation = "https://krishiv.ai/docs" +Benchmarks = "https://krishiv.ai/benchmarks" +Changelog = "https://github.com/KrishivAI/krishiv-engine/releases" +Source = "https://github.com/KrishivAI/krishiv-engine" + [project.optional-dependencies] -# Arrow libraries are optional: `__arrow_c_stream__` needs none (Polars and DuckDB consume it -# directly); pyarrow is needed by `toArrow`, which imports it lazily. `toPandas` needs the -# `pandas` extra (pandas and numpy; not bundled in the wheel). -pyarrow = ["pyarrow>=16"] +# pyarrow is a required dependency since 0.2.1; this extra stays so `krishiv[pyarrow]` install lines still work. +# `toPandas` needs the `pandas` extra (pandas and numpy; not bundled in the wheel). +pyarrow = [] pandas = ["pandas>=2.2", "numpy>=1.26", "pyarrow>=16"] [dependency-groups] diff --git a/python/tests/test_cloud_stores.py b/python/tests/test_cloud_stores.py new file mode 100644 index 000000000..b9663eea4 --- /dev/null +++ b/python/tests/test_cloud_stores.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Cloud object stores are built into the wheel (0.2.0 shipped without them: ``s3://``, ``gs://`` and ``az://`` +failed with KRISHIV_UNSUPPORTED_FEATURE). + +The offline test runs everywhere: a store URL must get past "not built in" and fail on the network instead. The round +trip runs when ``KRISHIV_TEST_S3_ENDPOINT`` names an S3 emulator with a bucket ``krishiv-test`` (MinIO, credentials +``minioadmin``/``minioadmin``; ``tests/scripts/object_store_emulators.sh`` and the wheels workflow start one).""" + +from __future__ import annotations + +import os +import uuid +from typing import Any + +import pytest + +from krishiv.sql import SparkSession + +ENDPOINT = os.environ.get("KRISHIV_TEST_S3_ENDPOINT") + + +@pytest.fixture +def spark() -> SparkSession: + return SparkSession.builder.create() + + +@pytest.mark.parametrize( + "url", + [ + "s3://krishiv-test/x.parquet", + "gs://krishiv-test/x.parquet", + "az://krishiv-test/x.parquet", + "hf://datasets/krishiv-test/x/x.parquet", + ], +) +def test_cloud_urls_are_built_into_the_wheel(spark: SparkSession, url: str) -> None: + # Point every store at a closed local port so nothing leaves the machine and the failure is quick. + for key, value in { + "fs.s3a.endpoint": "http://127.0.0.1:9", + "fs.s3a.access.key": "k", + "fs.s3a.secret.key": "s", + "fs.s3a.endpoint.region": "us-east-1", + "fs.s3a.connection.maximum": "1", + }.items(): + spark.conf.set(key, value) + with pytest.raises(Exception) as err: # noqa: PT011 - the class depends on the store; the text is what matters + spark.read.parquet(url).collect() + assert "KRISHIV_UNSUPPORTED_FEATURE" not in str(err.value), err.value + + +@pytest.mark.skipif(not ENDPOINT, reason="KRISHIV_TEST_S3_ENDPOINT is not set") +def test_s3_round_trip_against_an_emulator(spark: SparkSession) -> None: + pa = pytest.importorskip("pyarrow") # the offline test above must still run where pyarrow is absent (musllinux) + options: dict[str, Any] = { + "fs.s3a.endpoint": ENDPOINT, + "fs.s3a.access.key": "minioadmin", + "fs.s3a.secret.key": "minioadmin", + "fs.s3a.path.style.access": "true", + "fs.s3a.endpoint.region": "us-east-1", + } + for key, value in options.items(): + spark.conf.set(key, value) + url = f"s3://krishiv-test/wheel-{uuid.uuid4().hex}" + spark.createDataFrame(pa.table({"id": [3, 1, 2], "name": ["c", None, "b"]})).write.parquet(url) + rows = sorted((r["id"], r["name"]) for r in spark.read.parquet(url).collect()) + assert rows == [(1, None), (2, "b"), (3, "c")] diff --git a/python/uv.lock b/python/uv.lock index 8b4ebaff0..a346267b7 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -189,6 +189,9 @@ wheels = [ [[package]] name = "krishiv" source = { editable = "." } +dependencies = [ + { name = "pyarrow" }, +] [package.optional-dependencies] pandas = [ @@ -197,9 +200,6 @@ pandas = [ { name = "pandas" }, { name = "pyarrow" }, ] -pyarrow = [ - { name = "pyarrow" }, -] [package.dev-dependencies] dev = [ @@ -216,8 +216,8 @@ dev = [ requires-dist = [ { name = "numpy", marker = "extra == 'pandas'", specifier = ">=1.26" }, { name = "pandas", marker = "extra == 'pandas'", specifier = ">=2.2" }, + { name = "pyarrow", specifier = ">=16" }, { name = "pyarrow", marker = "extra == 'pandas'", specifier = ">=16" }, - { name = "pyarrow", marker = "extra == 'pyarrow'", specifier = ">=16" }, ] provides-extras = ["pandas", "pyarrow"]