Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions .github/workflows/ci-python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,44 @@ jobs:
-k 'TIMESTAMP or temporal'
done

# Released bindings lack Rust main features, so only native planning is forced here.
rust-release-plan:
name: Native CI / pypaimon-rust ${{ matrix.pypaimon-rust }}
timeout-minutes: 90
runs-on: ubuntu-latest
container: "python:3.11-slim"
strategy:
fail-fast: false
matrix:
pypaimon-rust: [ '0.4.0' ]
steps:
- name: Checkout code
uses: actions/checkout@v6

- name: Install system dependencies
run: |
apt-get update && apt-get install -y --no-install-recommends \
build-essential git curl cmake pkg-config libssl-dev libclang-dev tzdata \
&& rm -rf /var/lib/apt/lists/*

- name: Install Python test dependencies
run: |
python -m pip install --upgrade pip
python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
python -m pip install -e ./paimon-python --group ./paimon-python/pyproject.toml:dev \
'pypaimon-rust==${{ matrix.pypaimon-rust }}'
python -m pip check
python -c "from importlib.metadata import version; print('pypaimon-rust', version('pypaimon-rust'))"

- name: Run Python tests with native planning
shell: bash
env:
PYPAIMON_TEST_NATIVE_PLAN: '1'
# Tests marked native_plan need Rust main.
PYTEST_ADDOPTS: "-m 'not native_plan'"
run: |
bash paimon-python/dev/lint-python.sh -i pytest

torch:
name: PyTorch / Python 3.10
timeout-minutes: 60
Expand Down
4 changes: 3 additions & 1 deletion paimon-python/pypaimon/read/native_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -577,4 +577,6 @@ def _from_native_plan(table, rust_plan) -> Plan:
# Older bindings cannot distinguish an empty committed snapshot from a
# table without snapshots. Let the Python scanner recover the metadata.
raise RuntimeError("Native runtime cannot report an empty plan's snapshot")
return Plan(splits, snapshot_id=snapshot_id, _native_stats=rust_plan.stats())
# Bindings before Plan.stats() plan natively but report no counters.
stats = getattr(rust_plan, 'stats', None)
return Plan(splits, snapshot_id=snapshot_id, _native_stats=stats() if callable(stats) else None)
9 changes: 9 additions & 0 deletions paimon-python/pypaimon/tests/native_plan_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,15 @@ def test_native_final_statistics_preserve_work_and_saturate_unknown_sizes(self):
with patch.object(ScanStats, 'from_native', return_value=ScanStats.empty_native()):
self.assertEqual(zero.stats().planning_counters['limit'], 0)

def test_bindings_without_plan_stats_still_plan_natively(self):
from pypaimon.read.native_plan import _from_native_plan
table = Mock(table_schema=SimpleNamespace(fields=[], partition_keys=[]),
trimmed_primary_keys_fields=[])
table.options.native_read_enabled.return_value = False
plan = _from_native_plan(table, SimpleNamespace(splits=lambda: [], snapshot_id=lambda: 7))
self.assertEqual(plan.snapshot_id, 7)
self.assertIsNone(plan.stats())

def test_variant_read_type_matches_java_interop_fixture(self):
actual = _variant_read_type_json(
[DataField(7, 'payload', AtomicType('VARIANT'))],
Expand Down
Loading