Merge pull request #6 from sidney/issue-3

Issue 3

CHANGELOG.md

@@ -1,10 +1,15 @@
 # Changelog
 
-## [1.0.3] - \*\*put date here\*\*
+## [1.0.3] - \\put release date here\\
 
 ### Changed
 
 - Optimized read_symbol to read 16 bits at a time as is done in the C reference implementation
+- Fixed issue with compressed input that contains data split across multiple metadata blocks
+- Some changes to unit/integration tests to better support testing issues in compression format
+- Renamed test module to conform to pattern expected by unittest
+- Unit tests can now test using multiple compressed versions of each uncompressed example file 
+- Fixed error revealed by one of the newly tested cases involving an edge case in compressed format
 
 ## [1.0.2] - 2021-02-02
 

README.md

@@ -10,7 +10,7 @@ which it is not practical to package or require platform-specific binaries. It i
 the decompression function, under the assumption that will be the most common use-case that
 might have that restriction. It is hundreds of times slower than the reference `brotli`.
 
-This is a hand port of the decompression portion of the Javascript project that is
+This code began as a hand port of the decompression portion of the Javascript project that is
 itself a hand port of the C code of the reference implementation.
 
 * JavaScript port of brotli [brotli.js](https://github.com/devongovett/brotli.js)
@@ -28,12 +28,13 @@ that contains brotli compressed data, and return one with the uncompressed data
 
     uncompressed_data = decompress(compressed_data)
 
-### Running the integration tests
+### Running the unit and integration tests
 With a copy of the entire `test` directory, set PYTHONPATH to the directory containing
-`brotlidecpy.py`, which may or may not be the same directory that contains `test`,
-and run the command (suitably modified for the current directory you are using)
+`brotlidecpy.py` (optional if it is the current directory), which may or may not be the same
+directory that contains `test`, and run the command (suitably modified for the current
+directory you are using)
 
-    python test/decompresstest.py
+    python -m unittest discover test
 
 
 ###### brotlidecpy is open-sourced under the MIT License, see the LICENSE file.

brotlidecpy/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 
-__version__ = "1.0.2"
+__version__ = "1.0.3"
 
 # noinspection PyUnresolvedReferences
 from .decode import brotli_decompress_buffer as decompress

brotlidecpy/decode.py

@@ -354,16 +354,7 @@ def jump_to_byte_boundary(br):
 
 def brotli_decompress_buffer(input_buffer):
     br = BrotliBitReader(input_buffer)
-    decode_window_bits(br)
-    out = decode_meta_block_length(br)
-    decompressed_size = out.meta_block_length
-    output_buffer = bytearray([0] * decompressed_size)
-    br.reset()
-    brotli_decompress_br_to_buffer(br, output_buffer)
-    return output_buffer
-
-
-def brotli_decompress_br_to_buffer(br, output_buffer):
+    output_buffer = bytearray([])
     pos = 0
     input_end = 0
     max_distance = 0
@@ -407,6 +398,9 @@ def brotli_decompress_br_to_buffer(br, output_buffer):
         if meta_block_remaining_len == 0:
             continue
 
+        if len(output_buffer) < (pos + meta_block_remaining_len):
+            output_buffer.extend(bytearray([0] * meta_block_remaining_len))
+
         if is_uncompressed:
             copy_uncompressed_block_to_output(meta_block_remaining_len, pos, output_buffer, br)
             pos += meta_block_remaining_len
@@ -471,8 +465,8 @@ def brotli_decompress_br_to_buffer(br, output_buffer):
                 kInsertLengthPrefixCode[insert_code].nbits)
             copy_length = kCopyLengthPrefixCode[copy_code].offset + br.read_bits(
                 kCopyLengthPrefixCode[copy_code].nbits)
-            prev_byte1 = output_buffer[pos - 1]
-            prev_byte2 = output_buffer[pos - 2]
+            prev_byte1 = output_buffer[pos - 1] if pos > 0 else 0
+            prev_byte2 = output_buffer[pos - 2] if pos > 1 else 0
             for j in range(0, insert_length):
                 if block_length[0] == 0:
                     decode_block_type(num_block_types[0], block_type_trees, 0, block_type, block_type_rb,
@@ -557,7 +551,8 @@ def brotli_decompress_br_to_buffer(br, output_buffer):
                     raise Exception("Invalid backward reference. pos: %s distance: %s len: %s bytes left: %s" % (
                         pos, distance, copy_length, meta_block_remaining_len))
 
-                for j in range(0, copy_length):
+                for j in range(0, copy_length):  # don't try to optimize with a slice. source and dest may overlap
                     output_buffer[pos] = output_buffer[pos - distance]
                     pos += 1
                     meta_block_remaining_len -= 1
+    return output_buffer