diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8037922..8a481e4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,10 @@ version 1.0.1-dev + Fix ``gzip_ng_threaded`` writers at compression level 1, which raised ``OverflowError`` for data that does not compress and consists mostly of bytes 144 to 255. ++ Extraordinarily large gzip headers with either a very long comment or + filename will now cause an ``OverflowError`` rather than use more and more + memory. The size is limited to the internal buffer size which is currently + set at 512 KiB. version 1.0.0 ----------------- diff --git a/src/zlib_ng/zlib_ngmodule.c b/src/zlib_ng/zlib_ngmodule.c index c8c2a32..59233e5 100644 --- a/src/zlib_ng/zlib_ngmodule.c +++ b/src/zlib_ng/zlib_ngmodule.c @@ -2356,17 +2356,15 @@ GzipReader_read_from_file(GzipReader *self) const uint8_t *buffer_end = self->buffer_end; size_t remaining = buffer_end - current_pos; if (remaining == self->buffer_size) { - /* Buffer is full but a new read request was issued. This will be due - to the header being bigger than the header. Enlarge the buffer - to accommodate the hzip header. */ - size_t new_buffer_size = self->buffer_size * 2; - uint8_t *tmp_buffer = PyMem_Realloc(self->input_buffer, new_buffer_size); - if (tmp_buffer == NULL) { - PyErr_NoMemory(); - return -1; - } - self->input_buffer = tmp_buffer; - self->buffer_size = new_buffer_size; + /* Buffer is full but a new read request was issued. This will be due + to the header being bigger than the buffer. To prevent adverserial + inputs with infinite header sizes an error is thrown here. */ + PyErr_Format( + PyExc_OverflowError, + "Gzip header exceeds input buffer size of %lu", + self->buffer_size + ); + return -1; } else if (remaining > 0) { memmove(self->input_buffer, current_pos, remaining); } diff --git a/tests/test_gzip_ng.py b/tests/test_gzip_ng.py index 6c9d3c6..171bd84 100644 --- a/tests/test_gzip_ng.py +++ b/tests/test_gzip_ng.py @@ -14,6 +14,7 @@ import os import re import shutil +import struct import sys import tempfile import zlib @@ -409,7 +410,7 @@ def test_very_long_header_in_data(): def test_very_long_header_in_file(): # header with a very long filename. header = (b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff" + - gzip_ng.READ_BUFFER_SIZE * 2 * b"A" + + (gzip_ng.READ_BUFFER_SIZE - 11) * b"A" + b"\x00") compressed = header + zlib_ng.compress(b"", 3, -15) + 8 * b"\00" f = io.BytesIO(compressed) @@ -471,3 +472,27 @@ def test_bgzip(): with gzip_ng.open(gzip_file, "rb") as gz: gz_data = gz.read() assert bgz_data == gz_data + + +def test_adverserial_header(): + magic = b"\x1f\x8b" + method = 8 + mtime = 0 + flags = FCOMMENT + _os = 255 + xfl = 0 + header = struct.pack("<2sBBIBB", magic, method, flags, mtime, xfl, _os) + comment = b"A" * 10 * 1024 * 1024 + b"\00" + compressed = zlib_ng.compress(b"", wbits=-15) + crc = zlib_ng.crc32(b"") + length = 0 + trailer = struct.pack("