From 34c171bc51ce1c7dcb2e1e8cbb79a903afd7bd90 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Mon, 15 Sep 2025 08:14:04 +0200 Subject: [PATCH 01/20] Fix a spelling and copy paste error. Thanks @ellert --- tests/test_gzip_compliance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gzip_compliance.py b/tests/test_gzip_compliance.py index 8ad5441..64ea4b7 100644 --- a/tests/test_gzip_compliance.py +++ b/tests/test_gzip_compliance.py @@ -809,7 +809,7 @@ def test_decompress_stdin_stdout(self): # If the env_vars argument to assert_python_failure or assert_python_ok # is empty the test will run in isolated mode (-I) which means that the # PYTHONPATH environment variable will be ignored and the test fails - # because the isal module can not be found, or the test is run usung the + # because the zlib_ng module can not be found, or the test is run using the # system installed version of the module instead of the newly built # module that should be tested. # From 07c7f5f5ce0eee0290bf388bd1bfdd5c6cb30891 Mon Sep 17 00:00:00 2001 From: Finn Womack Date: Wed, 3 Dec 2025 01:48:13 -0800 Subject: [PATCH 02/20] Add windows arm64 wheelbuilding (#73) * add arm64 builds & tests, run workflow * skip cp39 and cp310 on arm64 windows * add v5 python-setup for arm64 windows Updated Python setup action for Windows ARM architecture. * fix typo * print env variables for windows to debug * fix pwsh command before build * run SET * Specify architecture for MSVC developer prompt * add arch option to msvc action * remove SET * uncomment skipped sections, remove fork branch trigger * update CHANGELOG.rst Added version 1.0.1-dev section with a note about Windows arm64 wheel support. --- .github/workflows/ci.yml | 15 +++++++++++++-- CHANGELOG.rst | 4 ++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2bc13d..e1ce7ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: python-version: "3.10" - os: "windows-latest" python-version: "3.10" + - os: "windows-11-arm" + python-version: "3.11" steps: - uses: actions/checkout@v4 with: @@ -83,6 +85,8 @@ jobs: - name: Set MSVC developer prompt uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' + with: + arch: ${{ matrix.os == 'windows-11-arm' && 'arm64' || 'x64' }} - name: Install build dependencies (MacOS) run: brew install make if: runner.os == 'macOS' @@ -170,6 +174,7 @@ jobs: - macos-13 - macos-latest - windows-latest + - windows-11-arm cibw_archs_linux: ["x86_64"] build_sdist: [true] include: @@ -180,7 +185,11 @@ jobs: with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - - uses: actions/setup-python@v2 # Some issues where caused by higher versions. + - if: ${{ matrix.os == 'windows-11-arm' }} + uses: actions/setup-python@v5 + name: Install Python + - if: ${{ matrix.os != 'windows-11-arm' }} + uses: actions/setup-python@v2 # Some issues where caused by higher versions. name: Install Python - name: Install cibuildwheel twine build run: python -m pip install cibuildwheel twine build @@ -190,6 +199,8 @@ jobs: - name: Set MSVC developer prompt uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' + with: + arch: ${{ matrix.os == 'windows-11-arm' && 'arm64' || 'x64' }} - name: Set up QEMU if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}} uses: docker/setup-qemu-action@v3 @@ -199,7 +210,7 @@ jobs: run: cibuildwheel --output-dir dist env: # Skip 32 bit, macosx_arm64 causes issues on cpython 3.9 - CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64" + CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64 cp39-win_arm64 cp310-win_arm64" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_TEST_REQUIRES: "pytest" # Simple tests that requires the project to be build correctly diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c067325..6357ca6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. +version 1.0.1-dev +----------------- ++ Wheels are now built for Windows arm64 architectures. + version 1.0.0 ----------------- The library has been running without issues as a dependency in quite a few From 70dafe550a59df3420cf22acf43ce9d65959819b Mon Sep 17 00:00:00 2001 From: Matt Stone Date: Wed, 20 May 2026 15:56:17 -0400 Subject: [PATCH 03/20] Re-enable PyPy wheel builds --- .github/workflows/ci.yml | 1 + CHANGELOG.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1ce7ff..0c527e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,6 +209,7 @@ jobs: - name: Build wheels run: cibuildwheel --output-dir dist env: + CIBW_ENABLE: "pypy" # Skip 32 bit, macosx_arm64 causes issues on cpython 3.9 CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_arm64 cp39-win_arm64 cp310-win_arm64" CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6357ca6..4d7eca3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ Changelog version 1.0.1-dev ----------------- + Wheels are now built for Windows arm64 architectures. ++ Restore PyPy wheel builds. version 1.0.0 ----------------- From 10d2ffd6e97dca906da84d59c5b39fad915b5262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:02:25 -0600 Subject: [PATCH 04/20] Fix a crash when calling ``copy()`` on a flushed compress object on Python 3.15 (#80) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix a crash when calling `copy()` on a flushed compress object on Python 3.15 Signed-off-by: Edgar Ramírez Mondragón * test: Address Pytest 9.1 warnings Signed-off-by: Edgar Ramírez Mondragón * chore: Allow usage of ` tox -e 3.15 -- ` Signed-off-by: Edgar Ramírez Mondragón * ci: Add Python 3.15 and 3.15t to CI matrix Signed-off-by: Edgar Ramírez Mondragón * ci: Replace `macos-13` runner with `macos-15-intel` Signed-off-by: Edgar Ramírez Mondragón --------- Signed-off-by: Edgar Ramírez Mondragón --- .github/workflows/ci.yml | 8 +++++--- CHANGELOG.rst | 5 +++++ src/zlib_ng/zlib_ngmodule.c | 3 ++- tests/test_compat.py | 24 ++++++++++++------------ tests/test_gzip_ng.py | 2 +- tests/test_gzip_ng_threaded.py | 4 ++-- tox.ini | 2 +- 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1ce7ff..f3954ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,13 +59,15 @@ jobs: - "3.13" - "3.14" - "3.14t" + - "3.15" + - "3.15t" - "pypy-3.10" - "pypy-3.11" os: ["ubuntu-latest"] include: - os: "macos-latest" # For m1 macos python-version: "3.12" - - os: "macos-13" # for x86 macos + - os: "macos-15-intel" # for x86 macos python-version: "3.10" - os: "windows-latest" python-version: "3.10" @@ -133,7 +135,7 @@ jobs: matrix: os: - "ubuntu-latest" - - "macos-13" + - "macos-15-intel" - "macos-latest" - "windows-latest" python_version: [ "python" ] @@ -171,7 +173,7 @@ jobs: matrix: os: - ubuntu-latest - - macos-13 + - macos-15-intel - macos-latest - windows-latest - windows-11-arm diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6357ca6..ca11abb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,11 @@ Changelog version 1.0.1-dev ----------------- + Wheels are now built for Windows arm64 architectures. ++ Fix a crash when calling ``copy()`` on a flushed compress object on Python + 3.15. Python 3.15 changed + (`gh-134745 `__) + ``PyThread_release_lock`` to use ``PyMutex`` internally, which now raises a + fatal error when releasing a lock that was never acquired. version 1.0.0 ----------------- diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c index ea71d53..c8c2a32 100644 --- a/src/zlib_ng/zlib_ngmodule.c +++ b/src/zlib_ng/zlib_ngmodule.c @@ -801,7 +801,8 @@ zlib_Compress_copy(compobject *self, PyObject *Py_UNUSED(ignored)) if (!self->is_initialised) { PyErr_SetString(PyExc_ValueError, "Cannot copy flushed objects."); - goto error; + Py_DECREF(return_value); + return NULL; } /* Copy the zstream state diff --git a/tests/test_compat.py b/tests/test_compat.py index 12dda97..871bf84 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -64,21 +64,21 @@ def limited_zlib_tests(strategies=ZLIB_STRATEGIES): @pytest.mark.parametrize(["data_size", "value"], - itertools.product(DATA_SIZES, SEEDS)) + list(itertools.product(DATA_SIZES, SEEDS))) def test_crc32(data_size, value): data = DATA[:data_size] assert zlib.crc32(data, value) == zlib_ng.crc32(data, value) @pytest.mark.parametrize(["data_size", "value"], - itertools.product(DATA_SIZES, SEEDS)) + list(itertools.product(DATA_SIZES, SEEDS))) def test_adler32(data_size, value): data = DATA[:data_size] assert zlib.adler32(data, value) == zlib_ng.adler32(data, value) @pytest.mark.parametrize(["data_size", "level", "wbits"], - itertools.product(DATA_SIZES, range(10), WBITS_RANGE)) + list(itertools.product(DATA_SIZES, range(10), WBITS_RANGE))) def test_compress(data_size, level, wbits): data = DATA[:data_size] compressed = zlib_ng.compress(data, level=level, wbits=wbits) @@ -87,7 +87,7 @@ def test_compress(data_size, level, wbits): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(10))) + list(itertools.product(DATA_SIZES, range(10)))) def test_decompress_zlib(data_size, level): data = DATA[:data_size] compressed = zlib.compress(data, level=level) @@ -96,7 +96,7 @@ def test_decompress_zlib(data_size, level): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel", "strategy"], - limited_zlib_tests(ZLIB_STRATEGIES)) + list(limited_zlib_tests(ZLIB_STRATEGIES))) def test_decompress_wbits(data_size, level, wbits, memLevel, strategy): data = DATA[:data_size] compressobj = zlib.compressobj(level=level, wbits=wbits, memLevel=memLevel, @@ -107,7 +107,7 @@ def test_decompress_wbits(data_size, level, wbits, memLevel, strategy): @pytest.mark.parametrize(["data_size", "level", "wbits"], - itertools.product([128 * 1024], range(10), WBITS_RANGE),) + list(itertools.product([128 * 1024], range(10), WBITS_RANGE),)) def test_decompress_zlib_ng(data_size, level, wbits): data = DATA[:data_size] compressed = zlib_ng.compress(data, level=level, wbits=wbits) @@ -116,7 +116,7 @@ def test_decompress_zlib_ng(data_size, level, wbits): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel", "strategy"], - limited_zlib_tests(ZLIBNG_STRATEGIES)) + list(limited_zlib_tests(ZLIBNG_STRATEGIES))) def test_compress_compressobj(data_size, level, wbits, memLevel, strategy): data = DATA[:data_size] compressobj = zlib_ng.compressobj(level=level, @@ -129,7 +129,7 @@ def test_compress_compressobj(data_size, level, wbits, memLevel, strategy): @pytest.mark.parametrize(["data_size", "level", "wbits", "memLevel", "strategy"], - limited_zlib_tests(ZLIB_STRATEGIES)) + list(limited_zlib_tests(ZLIB_STRATEGIES))) def test_decompress_decompressobj(data_size, level, wbits, memLevel, strategy): data = DATA[:data_size] compressobj = zlib.compressobj(level=level, wbits=wbits, memLevel=memLevel, @@ -151,7 +151,7 @@ def test_decompressobj_unconsumed_tail(): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(10))) + list(itertools.product(DATA_SIZES, range(10)))) def test_gzip_ng_compress(data_size, level): data = DATA[:data_size] compressed = gzip_ng.compress(data, compresslevel=level) @@ -159,7 +159,7 @@ def test_gzip_ng_compress(data_size, level): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(10))) + list(itertools.product(DATA_SIZES, range(10)))) def test_decompress_gzip(data_size, level): data = DATA[:data_size] compressed = gzip.compress(data, compresslevel=level) @@ -168,7 +168,7 @@ def test_decompress_gzip(data_size, level): @pytest.mark.parametrize(["data_size", "level"], - itertools.product(DATA_SIZES, range(10))) + list(itertools.product(DATA_SIZES, range(10)))) def test_decompress_gzip_ng(data_size, level): data = DATA[:data_size] compressed = gzip_ng.compress(data, compresslevel=level) @@ -177,7 +177,7 @@ def test_decompress_gzip_ng(data_size, level): @pytest.mark.parametrize(["unused_size", "wbits"], - itertools.product([26], [-15, 15, 31])) + list(itertools.product([26], [-15, 15, 31]))) def test_unused_data(unused_size, wbits): unused_data = b"abcdefghijklmnopqrstuvwxyz"[:unused_size] compressor = zlib.compressobj(wbits=wbits) diff --git a/tests/test_gzip_ng.py b/tests/test_gzip_ng.py index abfd283..6c9d3c6 100644 --- a/tests/test_gzip_ng.py +++ b/tests/test_gzip_ng.py @@ -65,7 +65,7 @@ def test_GzipNGFile_read_truncated(): "reached") -@pytest.mark.parametrize("level", range(1, 10)) +@pytest.mark.parametrize("level", list(range(1, 10))) def test_decompress_stdin_stdout(capsysbinary, level): """Test if the command line can decompress data that has been compressed by gzip at all levels.""" diff --git a/tests/test_gzip_ng_threaded.py b/tests/test_gzip_ng_threaded.py index b976419..a587193 100644 --- a/tests/test_gzip_ng_threaded.py +++ b/tests/test_gzip_ng_threaded.py @@ -31,7 +31,7 @@ def test_threaded_read(): @pytest.mark.parametrize(["mode", "threads"], - itertools.product(["wb", "wt"], [1, 3, -1])) + list(itertools.product(["wb", "wt"], [1, 3, -1]))) def test_threaded_write(mode, threads): with tempfile.NamedTemporaryFile("wb", delete=False) as tmp: # Use a small block size to simulate many writes. @@ -216,7 +216,7 @@ def test_threaded_writer_does_not_close_stream(): @pytest.mark.timeout(5) @pytest.mark.parametrize( - ["mode", "threads"], itertools.product(["rb", "wb"], [1, 2])) + ["mode", "threads"], list(itertools.product(["rb", "wb"], [1, 2]))) def test_threaded_program_can_exit_on_error(tmp_path, mode, threads): program = tmp_path / "no_context_manager.py" test_file = tmp_path / "output.gz" diff --git a/tox.ini b/tox.ini index 14409e4..dfefd90 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ setenv= PYTHONDEVMODE=1 commands = # Create HTML coverage report for humans and xml coverage report for external services. - coverage run --branch --source=zlib_ng -m pytest tests + coverage run --branch --source=zlib_ng -m pytest {posargs:tests} # Ignore errors during report generation. Pypy does not generate proper coverage reports. coverage html -i coverage xml -i From 06bcbeffcfd0de36102a4dd8ccdef79647378919 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 09:38:36 -0400 Subject: [PATCH 05/20] Fix gzip_ng_threaded writers at level 1 for bytes 144 to 255 In the bundled zlib-ng, level 1 uses deflate_quick(), which codes all data with the fixed Huffman codes and never falls back to stored blocks. Bytes 144 to 255 take 9 bits in these codes, so data made of them with no repeats grows by 12.5%. Each compressor had an output buffer only about 10% larger than the block size, so compress_and_crc() raised OverflowError. At level 1, make the buffer the block size plus an eighth plus 500 bytes, because no input byte takes more than 9 bits. --- CHANGELOG.rst | 3 +++ src/zlib_ng/gzip_ng_threaded.py | 11 ++++++++++- tests/test_gzip_ng_threaded.py | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ca11abb..0005ddd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,9 @@ version 1.0.1-dev (`gh-134745 `__) ``PyThread_release_lock`` to use ``PyMutex`` internally, which now raises a fatal error when releasing a lock that was never acquired. ++ Fix ``gzip_ng_threaded`` writers at compression level 1, which raised + ``OverflowError`` for data that does not compress and consists mostly of + bytes 144 to 255. version 1.0.0 ----------------- diff --git a/src/zlib_ng/gzip_ng_threaded.py b/src/zlib_ng/gzip_ng_threaded.py index dee8be7..b0e9fee 100644 --- a/src/zlib_ng/gzip_ng_threaded.py +++ b/src/zlib_ng/gzip_ng_threaded.py @@ -228,7 +228,16 @@ def __init__(self, self.previous_block = b"" # Deflating random data results in an output a little larger than the # input. Making the output buffer 10% larger is sufficient overkill. - compress_buffer_size = block_size + max(block_size // 10, 500) + # Level 1 is the exception: it codes all data with the fixed Huffman + # codes and never falls back to stored blocks. Bytes 144 to 255 take 9 + # bits in these codes, so data made of them grows by 12.5%. No byte + # takes more than 9 bits, so the block size plus an eighth plus 500 + # bytes for the block header, the end-of-block code, and the sync + # flush marker always fits. + if level == 1: + compress_buffer_size = block_size + block_size // 8 + 500 + else: + compress_buffer_size = block_size + max(block_size // 10, 500) self.block_size = block_size self.compressors: List[zlib_ng._ParallelCompress] = [ zlib_ng._ParallelCompress(buffersize=compress_buffer_size, diff --git a/tests/test_gzip_ng_threaded.py b/tests/test_gzip_ng_threaded.py index a587193..6aaa1b2 100644 --- a/tests/test_gzip_ng_threaded.py +++ b/tests/test_gzip_ng_threaded.py @@ -98,6 +98,24 @@ def test_threaded_write_oversized_block_no_error(threads): assert data == decompressed +@pytest.mark.timeout(5) +@pytest.mark.parametrize(["threads", "block_size"], + list(itertools.product([1, 3], + [8 * 1024, 1024 * 1024]))) +def test_threaded_write_high_bytes_level_1(tmp_path, threads, block_size): + # Level 1 codes all data with the fixed Huffman codes and never falls back + # to stored blocks. Bytes 144 to 255 take 9 bits in these codes, so data + # made of them grows by 12.5%. Map random bytes onto 144 to 255. + data = os.urandom(3 * block_size).translate( + bytes(144 + i % 112 for i in range(256))) + test_file = tmp_path / "output.gz" + with gzip_ng_threaded.open(test_file, "wb", compresslevel=1, + threads=threads, block_size=block_size) as f: + f.write(data) + with gzip.open(test_file, "rb") as f: + assert f.read() == data + + @pytest.mark.timeout(5) @pytest.mark.parametrize("threads", [1, 3]) def test_threaded_write_error(threads): From 42384b132da4d6e160f24b390f6dd74a8548f79c Mon Sep 17 00:00:00 2001 From: ptweezy Date: Tue, 20 Oct 2026 09:57:09 -0400 Subject: [PATCH 06/20] Raise write errors in gzip_ng_threaded instead of waiting forever When raw.write() raised an error in the thread that writes the compressed blocks, that thread exited without calling task_done(). As a result, flush() waited forever, and so did closing the file object that gzip_ng_threaded.open() returns. write() blocked forever when an input queue was full. The thread now stores the error and drops later blocks until stop() ends it, so write(), flush(), and close() can raise the error in the calling thread. After an error, close() closes raw only if closefd is true, so it no longer closes a file object passed to gzip_ng_threaded.open(). close() now calls flush() before it writes the trailer, as python-isal's close() has done since pycompression/python-isal@5ef868b, so the trailer always comes after the queued blocks. This ports pycompression/python-isal#275. --- CHANGELOG.rst | 5 ++ src/zlib_ng/gzip_ng_threaded.py | 48 ++++++++++++----- tests/test_gzip_ng_threaded.py | 92 +++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ca11abb..854ea04 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,11 @@ Changelog version 1.0.1-dev ----------------- ++ Fix ``gzip_ng_threaded`` writers that waited forever when writing to the + underlying file failed, for example, because the disk was full. A later call + to ``write()``, ``flush()``, or ``close()`` raises the error. After an error, + ``close()`` no longer closes a file object that you passed to + ``gzip_ng_threaded.open``. + Wheels are now built for Windows arm64 architectures. + Fix a crash when calling ``copy()`` on a flushed compress object on Python 3.15. Python 3.15 changed diff --git a/src/zlib_ng/gzip_ng_threaded.py b/src/zlib_ng/gzip_ng_threaded.py index dee8be7..934a8fb 100644 --- a/src/zlib_ng/gzip_ng_threaded.py +++ b/src/zlib_ng/gzip_ng_threaded.py @@ -329,25 +329,32 @@ def flush(self): # Wait for all data to be written for out_q in self.output_queues: out_q.join() + if self.exception: + raise self.exception self.raw.flush() def close(self): if self._closed: return - self.raw.write(zlib_ng.compress(b"", wbits=-15)) - trailer = struct.pack(" bool: @@ -388,7 +395,7 @@ def _write(self): continue self._crc = zlib_ng.crc32_combine(self._crc, crc, data_length) self._size += data_length - self.raw.write(compressed) + self._write_compressed(compressed) output_queue.task_done() index += 1 @@ -413,9 +420,22 @@ def _compress_and_write(self): data_length = len(data) self._crc = zlib_ng.crc32_combine(self._crc, crc, data_length) self._size += data_length - self.raw.write(compressed) + self._write_compressed(compressed) in_queue.task_done() + def _write_compressed(self, compressed: bytes): + # After a write error, this thread keeps taking blocks from the queues + # and drops them until stop() ends it. That way, no thread waits + # forever on a full queue, and write(), flush(), and close() can raise + # the error in the calling thread. + if self.exception: + return + try: + self.raw.write(compressed) + except Exception as e: + with self.lock: + self.exception = e + def _set_error_and_empty_queue(self, error, q): with self.lock: self.exception = error diff --git a/tests/test_gzip_ng_threaded.py b/tests/test_gzip_ng_threaded.py index a587193..0ede0c0 100644 --- a/tests/test_gzip_ng_threaded.py +++ b/tests/test_gzip_ng_threaded.py @@ -5,6 +5,7 @@ # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. +import errno import gzip import io import itertools @@ -12,6 +13,8 @@ import subprocess import sys import tempfile +import time +import types import zlib from pathlib import Path @@ -112,6 +115,83 @@ def test_threaded_write_error(threads): error.match("Compressed output exceeds buffer size") +class FullDiskStream(io.BytesIO): + """Accepts the 10-byte gzip header, then fails like a full disk.""" + before_error = None + + def write(self, b): + if self.tell() == 0: + return super().write(b) + if self.before_error: + self.before_error() + raise OSError(errno.ENOSPC, os.strerror(errno.ENOSPC)) + + +def assert_writer_closed(f): + assert f.closed + workers = [f.output_worker] + f.compression_workers + assert not any(worker.is_alive() for worker in workers) + # Make sure double closing does not raise errors + f.close() + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("threads", [1, 3]) +def test_threaded_write_error_with_full_queues(threads): + raw = FullDiskStream() + f = gzip_ng_threaded._ThreadedGzipWriter(raw, threads=threads) + queues = f.input_queues + f.output_queues + + def wait_until_queues_are_full(): + while not all(q.full() for q in queues): + time.sleep(0.001) + + # Fail when all queues are full, so write() usually waits for room in a + # full input queue. + raw.before_error = wait_until_queues_are_full + with pytest.raises(OSError) as error: + while True: + f.write(b"x") + assert error.value.errno == errno.ENOSPC + with pytest.raises(OSError): + f.close() + assert_writer_closed(f) + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("threads", [1, 3]) +def test_threaded_flush_error(threads): + raw = FullDiskStream() + f = gzip_ng_threaded.open(raw, "wb", threads=threads) + f.write(b"x") + with pytest.raises(OSError) as error: + f.flush() + assert error.value.errno == errno.ENOSPC + # Later calls raise the error again. + with pytest.raises(OSError): + f.flush() + with pytest.raises(OSError): + f.close() + assert_writer_closed(f.raw) + # The test passed the stream to open(), so close() leaves it open. + assert not raw.closed + + +@pytest.mark.timeout(5) +@pytest.mark.parametrize("threads", [1, 3]) +def test_threaded_close_error(monkeypatch, threads): + raw = FullDiskStream() + # Make open() treat raw as a file that it opened by name. + monkeypatch.setattr(gzip_ng_threaded, "builtins", + types.SimpleNamespace(open=lambda name, mode: raw)) + with pytest.raises(OSError) as error: + with gzip_ng_threaded.open("full.gz", "wb", threads=threads) as f: + f.write(b"x") + assert error.value.errno == errno.ENOSPC + assert_writer_closed(f.raw) + assert raw.closed + + def test_close_reader(): tmp = io.BytesIO(Path(TEST_FILE).read_bytes()) f = gzip_ng_threaded._ThreadedGzipReader(tmp, "rb") @@ -131,6 +211,18 @@ def test_close_writer(threads): f.close() +@pytest.mark.parametrize("threads", [1, 3]) +def test_close_writer_writes_queued_blocks(threads): + raw = io.BytesIO() + f = gzip_ng_threaded._ThreadedGzipWriter( + raw, threads=threads, block_size=8 * 1024) + data = os.urandom(64 * 1024) + f.write(data) + # close() must write the queued blocks before the gzip trailer. + f.close() + assert gzip.decompress(raw.getvalue()) == data + + def test_reader_not_writable(): with gzip_ng_threaded.open(TEST_FILE, "rb") as f: assert not f.writable() From bb8611fa64876db165b4c0bc25b228c991ad420c Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Oct 2026 11:47:00 +0200 Subject: [PATCH 07/20] Update to latest actions --- .github/workflows/ci.yml | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d293a4a..7c0da01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,11 +16,11 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: recursive - name: Set up Python 3.10 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: "3.10" - name: Install tox @@ -36,11 +36,11 @@ jobs: - twine_check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: recursive - name: Set up Python 3.10 - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: "3.10" - name: Install tox and upgrade setuptools and pip @@ -74,11 +74,11 @@ jobs: - os: "windows-11-arm" python-version: "3.11" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} allow-prereleases: true @@ -95,7 +95,7 @@ jobs: - name: Run tests run: tox - name: Upload coverage report - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v7 test-arch: if: startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' @@ -105,7 +105,7 @@ jobs: python_version: - "3.10" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: recursive - uses: uraimo/run-on-arch-action@v3 @@ -143,11 +143,11 @@ jobs: - os: "ubuntu-latest" python_version: "pypy" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: recursive - name: Install miniconda. - uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda. + uses: conda-incubator/setup-miniconda@v4 # https://github.com/conda-incubator/setup-miniconda. with: channels: conda-forge,defaults - name: Install requirements (universal) @@ -183,15 +183,11 @@ jobs: - os: "ubuntu-latest" cibw_archs_linux: "aarch64" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - - if: ${{ matrix.os == 'windows-11-arm' }} - uses: actions/setup-python@v5 - name: Install Python - - if: ${{ matrix.os != 'windows-11-arm' }} - uses: actions/setup-python@v2 # Some issues where caused by higher versions. + - uses: actions/setup-python@v7 name: Install Python - name: Install cibuildwheel twine build run: python -m pip install cibuildwheel twine build @@ -205,7 +201,7 @@ jobs: arch: ${{ matrix.os == 'windows-11-arm' && 'arm64' || 'x64' }} - name: Set up QEMU if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}} - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 with: platforms: arm64 - name: Build wheels @@ -245,7 +241,7 @@ jobs: if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: python -m build --sdist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: "dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}" path: "dist/" From 7b6c2a271e1f246aaabb0dc984e43307c6993569 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Oct 2026 14:12:27 +0200 Subject: [PATCH 08/20] Test CI with supported version 3.11 --- .github/workflows/ci.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c0da01..59067c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,10 +19,10 @@ jobs: - uses: actions/checkout@v7 with: submodules: recursive - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v7 with: - python-version: "3.10" + python-version: "3.11" - name: Install tox run: pip install tox - name: Lint @@ -39,10 +39,10 @@ jobs: - uses: actions/checkout@v7 with: submodules: recursive - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v7 with: - python-version: "3.10" + python-version: "3.11" - name: Install tox and upgrade setuptools and pip run: pip install --upgrade tox setuptools pip - name: Run tox -e ${{ matrix.tox_env }} @@ -53,7 +53,6 @@ jobs: strategy: matrix: python-version: - - "3.10" - "3.11" - "3.12" - "3.13" @@ -61,16 +60,16 @@ jobs: - "3.14t" - "3.15" - "3.15t" - - "pypy-3.10" + - "pypy-3.11" - "pypy-3.11" os: ["ubuntu-latest"] include: - os: "macos-latest" # For m1 macos python-version: "3.12" - os: "macos-15-intel" # for x86 macos - python-version: "3.10" + python-version: "3.11" - os: "windows-latest" - python-version: "3.10" + python-version: "3.11" - os: "windows-11-arm" python-version: "3.11" steps: @@ -103,7 +102,7 @@ jobs: strategy: matrix: python_version: - - "3.10" + - "3.11" steps: - uses: actions/checkout@v7 with: From 1f3bcfd4522b0cd77d8830b1c32226c1b5f0e4a3 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Wed, 21 Oct 2026 14:18:14 +0200 Subject: [PATCH 09/20] Explicitly use python 3.11 for dynamic testing. --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59067c3..abb29d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -137,10 +137,7 @@ jobs: - "macos-15-intel" - "macos-latest" - "windows-latest" - python_version: [ "python" ] - include: - - os: "ubuntu-latest" - python_version: "pypy" + python_version: [ "python==3.11" ] steps: - uses: actions/checkout@v7 with: From 8998171166c3a6c27c9ce20eff54b4a89ab9d772 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Wed, 21 Oct 2026 19:45:54 -0400 Subject: [PATCH 10/20] Use the level 1 output buffer size at every level Review feedback: one buffer size for all levels is simpler, and the extra 2.5% of headroom at the other levels doesn't matter. --- src/zlib_ng/gzip_ng_threaded.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/zlib_ng/gzip_ng_threaded.py b/src/zlib_ng/gzip_ng_threaded.py index b0e9fee..172c20e 100644 --- a/src/zlib_ng/gzip_ng_threaded.py +++ b/src/zlib_ng/gzip_ng_threaded.py @@ -226,18 +226,9 @@ def __init__(self, self.exception: Optional[Exception] = None self.level = level self.previous_block = b"" - # Deflating random data results in an output a little larger than the - # input. Making the output buffer 10% larger is sufficient overkill. - # Level 1 is the exception: it codes all data with the fixed Huffman - # codes and never falls back to stored blocks. Bytes 144 to 255 take 9 - # bits in these codes, so data made of them grows by 12.5%. No byte - # takes more than 9 bits, so the block size plus an eighth plus 500 - # bytes for the block header, the end-of-block code, and the sync - # flush marker always fits. - if level == 1: - compress_buffer_size = block_size + block_size // 8 + 500 - else: - compress_buffer_size = block_size + max(block_size // 10, 500) + # Level 1 never falls back to stored blocks and codes bytes 144 to 255 + # with 9 bits each, so its output can be an eighth larger. + compress_buffer_size = block_size + block_size // 8 + 500 self.block_size = block_size self.compressors: List[zlib_ng._ParallelCompress] = [ zlib_ng._ParallelCompress(buffersize=compress_buffer_size, From 615081d4ec8f5f253e59172c1c92152373603eb5 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 10:17:33 +0100 Subject: [PATCH 11/20] Throw an overflowerror on adverserial headers --- CHANGELOG.rst | 4 ++++ src/zlib_ng/zlib_ngmodule.c | 20 +++++++++----------- tests/test_gzip_ng.py | 27 ++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8037922..8a481e4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,10 @@ version 1.0.1-dev + Fix ``gzip_ng_threaded`` writers at compression level 1, which raised ``OverflowError`` for data that does not compress and consists mostly of bytes 144 to 255. ++ Extraordinarily large gzip headers with either a very long comment or + filename will now cause an ``OverflowError`` rather than use more and more + memory. The size is limited to the internal buffer size which is currently + set at 512 KiB. version 1.0.0 ----------------- diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c index c8c2a32..59233e5 100644 --- a/src/zlib_ng/zlib_ngmodule.c +++ b/src/zlib_ng/zlib_ngmodule.c @@ -2356,17 +2356,15 @@ GzipReader_read_from_file(GzipReader *self) const uint8_t *buffer_end = self->buffer_end; size_t remaining = buffer_end - current_pos; if (remaining == self->buffer_size) { - /* Buffer is full but a new read request was issued. This will be due - to the header being bigger than the header. Enlarge the buffer - to accommodate the hzip header. */ - size_t new_buffer_size = self->buffer_size * 2; - uint8_t *tmp_buffer = PyMem_Realloc(self->input_buffer, new_buffer_size); - if (tmp_buffer == NULL) { - PyErr_NoMemory(); - return -1; - } - self->input_buffer = tmp_buffer; - self->buffer_size = new_buffer_size; + /* Buffer is full but a new read request was issued. This will be due + to the header being bigger than the buffer. To prevent adverserial + inputs with infinite header sizes an error is thrown here. */ + PyErr_Format( + PyExc_OverflowError, + "Gzip header exceeds input buffer size of %lu", + self->buffer_size + ); + return -1; } else if (remaining > 0) { memmove(self->input_buffer, current_pos, remaining); } diff --git a/tests/test_gzip_ng.py b/tests/test_gzip_ng.py index 6c9d3c6..171bd84 100644 --- a/tests/test_gzip_ng.py +++ b/tests/test_gzip_ng.py @@ -14,6 +14,7 @@ import os import re import shutil +import struct import sys import tempfile import zlib @@ -409,7 +410,7 @@ def test_very_long_header_in_data(): def test_very_long_header_in_file(): # header with a very long filename. header = (b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff" + - gzip_ng.READ_BUFFER_SIZE * 2 * b"A" + + (gzip_ng.READ_BUFFER_SIZE - 11) * b"A" + b"\x00") compressed = header + zlib_ng.compress(b"", 3, -15) + 8 * b"\00" f = io.BytesIO(compressed) @@ -471,3 +472,27 @@ def test_bgzip(): with gzip_ng.open(gzip_file, "rb") as gz: gz_data = gz.read() assert bgz_data == gz_data + + +def test_adverserial_header(): + magic = b"\x1f\x8b" + method = 8 + mtime = 0 + flags = FCOMMENT + _os = 255 + xfl = 0 + header = struct.pack("<2sBBIBB", magic, method, flags, mtime, xfl, _os) + comment = b"A" * 10 * 1024 * 1024 + b"\00" + compressed = zlib_ng.compress(b"", wbits=-15) + crc = zlib_ng.crc32(b"") + length = 0 + trailer = struct.pack(" Date: Tue, 3 Nov 2026 10:52:46 +0100 Subject: [PATCH 12/20] Update included zlib-ng distribution to 2.3.3. --- CHANGELOG.rst | 1 + src/zlib_ng/zlib-ng | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8a481e4..e30b5ea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Changelog version 1.0.1-dev ----------------- ++ Update included zlib-ng distribution to 2.3.3. + Fix ``gzip_ng_threaded`` writers that waited forever when writing to the underlying file failed, for example, because the disk was full. A later call to ``write()``, ``flush()``, or ``close()`` raises the error. After an error, diff --git a/src/zlib_ng/zlib-ng b/src/zlib_ng/zlib-ng index 4254390..1273109 160000 --- a/src/zlib_ng/zlib-ng +++ b/src/zlib_ng/zlib-ng @@ -1 +1 @@ -Subproject commit 425439062b114a0f6cf625022c41d929c7e879f9 +Subproject commit 12731092979c6d07f42da27da673a9f6c7b13586 From cba2a3ec766164b71ecc1d386c10d44629435232 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 11:44:21 +0100 Subject: [PATCH 13/20] Use Cmake always --- .github/workflows/ci.yml | 2 +- setup.py | 20 ++++++-------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb29d4..2157921 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: with: arch: ${{ matrix.os == 'windows-11-arm' && 'arm64' || 'x64' }} - name: Install build dependencies (MacOS) - run: brew install make + run: brew install cmake if: runner.os == 'macOS' - name: Run tests run: tox diff --git a/setup.py b/setup.py index f58cbf8..6884e85 100644 --- a/setup.py +++ b/setup.py @@ -103,20 +103,12 @@ def build_zlib_ng(): build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fPIC" # Add -fPIC flag to allow static compilation run_args = dict(cwd=build_dir, env=build_env) - if sys.platform == "darwin": # Cmake does not work properly - subprocess.run([os.path.join(build_dir, "configure")], **run_args) - make_program = "gmake" if shutil.which("gmake") else "make" - subprocess.run([make_program, "libz-ng.a"], **run_args) - elif sys.platform == "linux": - subprocess.run([os.path.join(build_dir, "configure")], **run_args) - subprocess.run(["make", "libz-ng.a", "-j", str(cpu_count)], **run_args) - else: - subprocess.run(["cmake", build_dir], **run_args) - # Do not create test suite and do not perform tests to shorten build times. - # There is no need when stable releases of zlib-ng are used. - subprocess.run(["cmake", "--build", build_dir, "--config", "Release", - "--target", "zlibstatic", - "-j", str(cpu_count)], **run_args) + subprocess.run(["cmake", build_dir], **run_args) + # Do not create test suite and do not perform tests to shorten build times. + # There is no need when stable releases of zlib-ng are used. + subprocess.run(["cmake", "--build", build_dir, "--config", "Release", + "--target", "zlib-ng-static", + "-j", str(cpu_count)], **run_args) if BUILD_CACHE: BUILD_CACHE_FILE.write_text(build_dir) return build_dir From 2f87f7fed57313b93b37612dd5028ef4bf2ddc92 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 11:46:32 +0100 Subject: [PATCH 14/20] Build macos with make --- .github/workflows/ci.yml | 2 +- setup.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2157921..abb29d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,7 +89,7 @@ jobs: with: arch: ${{ matrix.os == 'windows-11-arm' && 'arm64' || 'x64' }} - name: Install build dependencies (MacOS) - run: brew install cmake + run: brew install make if: runner.os == 'macOS' - name: Run tests run: tox diff --git a/setup.py b/setup.py index 6884e85..6d9f267 100644 --- a/setup.py +++ b/setup.py @@ -103,12 +103,17 @@ def build_zlib_ng(): build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fPIC" # Add -fPIC flag to allow static compilation run_args = dict(cwd=build_dir, env=build_env) - subprocess.run(["cmake", build_dir], **run_args) - # Do not create test suite and do not perform tests to shorten build times. - # There is no need when stable releases of zlib-ng are used. - subprocess.run(["cmake", "--build", build_dir, "--config", "Release", - "--target", "zlib-ng-static", - "-j", str(cpu_count)], **run_args) + if sys.platform == "darwin": # Cmake does not work properly + subprocess.run([os.path.join(build_dir, "configure")], **run_args) + make_program = "gmake" if shutil.which("gmake") else "make" + subprocess.run([make_program, "libz-ng.a"], **run_args) + else: + subprocess.run(["cmake", build_dir], **run_args) + # Do not create test suite and do not perform tests to shorten build times. + # There is no need when stable releases of zlib-ng are used. + subprocess.run(["cmake", "--build", build_dir, "--config", "Release", + "--target", "zlib-ng-static", + "-j", str(cpu_count)], **run_args) if BUILD_CACHE: BUILD_CACHE_FILE.write_text(build_dir) return build_dir From acbc3a55583039e415388e5664d9e96c368345f1 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 12:01:47 +0100 Subject: [PATCH 15/20] Publish to PyPI according to PyPA recommended method --- .github/workflows/ci.yml | 66 +++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb29d4..6ce4b05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,7 +156,31 @@ jobs: env: PYTHON_ZLIB_NG_LINK_DYNAMIC: True - deploy: + make_sdist: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + needs: + - lint + - package-checks + - test-static + - test-dynamic + - test-arch + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 # Optional, use if you use setuptools_scm + submodules: true # Optional, use if you have submodules + persist-credentials: false # Highly recommended as a good security practice to not store credentials in disk, unless you need this for your specific use case + + - name: Build SDist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: cibw-sdist + path: dist/*.tar.gz + + build_wheels: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') runs-on: ${{ matrix.os }} needs: @@ -239,19 +263,33 @@ jobs: python -m build --sdist - uses: actions/upload-artifact@v7 with: - name: "dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}" + name: "cibw-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}" path: "dist/" - - name: Publish package to TestPyPI - # pypa/gh-action-pypi-publish@master does not work on OSX + + upload_to_pypi: + needs: [ build_wheels, make_sdist ] + environment: pypi + permissions: + id-token: write + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + steps: + - uses: actions/download-artifact@v5 + with: + pattern: cibw-* + path: dist + merge-multiple: true + + - name: Publish to test PyPI # Alpha, Beta and dev releases contain a - in the tag. - if: contains(github.ref, '-') && startsWith(github.ref, 'refs/tags') - run: twine upload --skip-existing -r testpypi dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - - name: Publish package to PyPI + if: "contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')" + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + + - name: Publish to PyPI if: "!contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')" - run: twine upload --skip-existing dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true From c1ca45cff61d2d87d917bccf9f5b791d14c8f56b Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 12:03:11 +0100 Subject: [PATCH 16/20] Add sanitizer testing --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ce4b05..275fc72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,6 +156,49 @@ jobs: env: PYTHON_ZLIB_NG_LINK_DYNAMIC: True + # Run the tests on CPython builds with AddressSanitizer and ThreadSanitizer + # from https://github.com/nascheme/cpython_sanity. + test-sanitizers: + runs-on: ubuntu-latest + # A crash under ThreadSanitizer can hang instead of exiting. + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - sanitizer: address + image: ghcr.io/nascheme/cpython-asan:3.14 + # The free-threaded build runs Python threads in parallel. + - sanitizer: thread + image: ghcr.io/nascheme/cpython-tsan:3.14t + container: + image: ${{ matrix.image }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install build dependencies + run: apt-get update && apt-get install -y nasm + - name: Install python-isal with ${{ matrix.sanitizer }} sanitizer + # setuptools-scm needs git, and the container user does not own the + # repository. + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + python -m pip install . pytest pytest-timeout + env: + # The extension modules get -fsanitize from CPython. CFLAGS adds it + # to the C code of ISA-L. + CFLAGS: -fsanitize=${{ matrix.sanitizer }} -fno-omit-frame-pointer -g + - name: Run tests + # Do not set PYTHONDEVMODE. Its debug allocator pads each allocation, + # so AddressSanitizer misses small overflows. --capture=sys prints + # sanitizer reports from C code even when they end the process, and + # -v prints the name of the test before its report. + run: python -m pytest -v tests --capture=sys + env: + # Exit at the first data race. A crash after a race can hang. + TSAN_OPTIONS: halt_on_error=1 + make_sdist: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') needs: @@ -164,6 +207,7 @@ jobs: - test-static - test-dynamic - test-arch + - test-sanitizers runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -189,6 +233,7 @@ jobs: - test-static - test-dynamic - test-arch + - test-sanitizers strategy: matrix: os: From 1e9f1626dc5c7a393e891999d07b19abda9dc2b4 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 12:53:09 +0100 Subject: [PATCH 17/20] Ensure cmake is installed on sanitizer --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 275fc72..ec279fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,7 +178,7 @@ jobs: with: submodules: recursive - name: Install build dependencies - run: apt-get update && apt-get install -y nasm + run: apt-get update && apt-get install -y cmake - name: Install python-isal with ${{ matrix.sanitizer }} sanitizer # setuptools-scm needs git, and the container user does not own the # repository. From 674f35e3d236b352ebb45c8b8446474240e033dc Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 13:08:03 +0100 Subject: [PATCH 18/20] Fix broken pipe error when piping into programs using the CLI. --- CHANGELOG.rst | 2 ++ src/zlib_ng/gzip_ng.py | 3 +++ tests/test_gzip_ng.py | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e30b5ea..0bf5722 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,8 @@ Changelog version 1.0.1-dev ----------------- + Update included zlib-ng distribution to 2.3.3. ++ Fix BrokenPipeError when piping ``python -m isal.igzip`` into ``head``, + ``less`` or similar programs. + Fix ``gzip_ng_threaded`` writers that waited forever when writing to the underlying file failed, for example, because the disk was full. A later call to ``write()``, ``flush()``, or ``close()`` raises the error. After an error, diff --git a/src/zlib_ng/gzip_ng.py b/src/zlib_ng/gzip_ng.py index f24bdbe..c2afec2 100644 --- a/src/zlib_ng/gzip_ng.py +++ b/src/zlib_ng/gzip_ng.py @@ -304,6 +304,9 @@ def main(): READ_BUFFER_SIZE = args.buffer_size try: shutil.copyfileobj(in_file, out_file, args.buffer_size) + except BrokenPipeError: + # Do not throw errors when a pipe to head or less closes. + pass finally: if in_file is not sys.stdin.buffer: in_file.close() diff --git a/tests/test_gzip_ng.py b/tests/test_gzip_ng.py index 171bd84..41eb954 100644 --- a/tests/test_gzip_ng.py +++ b/tests/test_gzip_ng.py @@ -15,6 +15,7 @@ import re import shutil import struct +import subprocess import sys import tempfile import zlib @@ -496,3 +497,14 @@ def test_adverserial_header(): gzip_file.read() finally: os.remove(tmpfile.name) + + +@pytest.mark.skipif(sys.platform != "linux", reason="bash required") +def test_no_broken_pipe(): + subprocess.run( + [ + "bash", "-c", "-e", "-o", "pipefail", + f"{sys.executable} -m zlib_ng.gzip_ng -cd {TEST_FILE} | head -n 10" + ], + check=True + ) From c55f942ae536b08be9a3a08bbbd82ceeb5b1029f Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 16:19:33 +0100 Subject: [PATCH 19/20] Add cmake to readthedocs.yml --- .readthedocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index cb71cd8..1a83c66 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -14,6 +14,8 @@ build: os: "ubuntu-22.04" tools: python: "3" + apt_packages: + - cmake # Make sure zlib-ng is cloned alongside submodules: From a2d21760e45d1f5079a8bdb7436297a79a997ae5 Mon Sep 17 00:00:00 2001 From: Ruben Vorderman Date: Tue, 3 Nov 2026 16:24:10 +0100 Subject: [PATCH 20/20] Set version to 1.1.0 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0bf5722..11ec8d1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Changelog .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. -version 1.0.1-dev +version 1.1.0 ----------------- + Update included zlib-ng distribution to 2.3.3. + Fix BrokenPipeError when piping ``python -m isal.igzip`` into ``head``,