From 68f62768c6cb692a2ab9a1b5f48d731af90b7028 Mon Sep 17 00:00:00 2001 From: Thor Whalen <1906276+thorwhalen@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:59:07 +0100 Subject: [PATCH 1/3] wav_chunks: the public RIFF chunk walker; fast known-id scan for overrunning data (voxy#85) --- recode/__init__.py | 1 + recode/audio.py | 60 ++++++++++++++++++++++++++++++++++++++++++++-- test_recode.py | 45 ++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 2 deletions(-) diff --git a/recode/__init__.py b/recode/__init__.py index 992f3f7..b48b577 100644 --- a/recode/__init__.py +++ b/recode/__init__.py @@ -107,6 +107,7 @@ from recode.audio import ( encode_wav_bytes, decode_wav_bytes, + wav_chunks, encode_wav_header_bytes, decode_wav_header_bytes, mk_pcm_audio_codec, diff --git a/recode/audio.py b/recode/audio.py index 43660f5..f28e4d2 100644 --- a/recode/audio.py +++ b/recode/audio.py @@ -37,6 +37,7 @@ """ import operator +import re import struct import warnings import wave @@ -140,6 +141,26 @@ def decode_pcm_bytes(pcm_bytes: bytes, width: Width = 2, n_channels: int = 1): _SUBFORMAT_GUID_OFFSET = 24 # within the `fmt ` chunk contents # The SubFormat GUID that says an EXTENSIBLE chunk really does carry plain PCM. _PCM_SUBFORMAT_GUID = bytes.fromhex("0100000000001000800000aa00389b71") +# Where a chunk id (four printable ASCII characters) starts, as a zero-width match. +_CHUNK_ID_AHEAD = re.compile(rb"(?=[\x20-\x7e]{4})") +# ... and the ids encoders append after the audio (metadata, cues, padding). +KNOWN_TRAILING_CHUNK_IDS = ( + b"LIST", b"id3 ", b"ID3 ", b"fact", b"cue ", b"smpl", b"bext", b"JUNK", + b"PAD ", b"iXML", b"_PMX", b"acid", b"inst", b"plst", +) # fmt: skip + + +def _known_chunk_id_positions(wav_bytes, start: int, stop: int): + """Where a known trailing chunk id occurs in ``wav_bytes[start:stop]``, in order + (``bytes.find`` scans in C, far faster than a regex alternation).""" + found = [] + for chunk_id in KNOWN_TRAILING_CHUNK_IDS: + at = wav_bytes.find(chunk_id, start, stop + _CHUNK_ID_SIZE) + while at != -1: + found.append(at) + at = wav_bytes.find(chunk_id, at + 1, stop + _CHUNK_ID_SIZE) + return sorted(found) + class ShortWavData(UserWarning): @@ -336,6 +357,36 @@ def _wav_data_chunk(wav_bytes: bytes) -> tuple: return contents, _payload_end_of_overrunning_data(wav_bytes, contents) +def wav_chunks(wav_bytes: bytes): + r"""``(chunk_id, offset, size)`` of every chunk of a RIFF/WAVE buffer, in order. + + The one chunk walker: :func:`decode_wav_bytes` and other readers (voxy's numpy + one) take their ``fmt `` and ``data`` from it. ``offset`` and ``size`` locate the + chunk's contents. A ``data`` chunk whose declared size is a streaming placeholder + (``0``, or past the end of the buffer) ends where the chunks after it begin, or + at the end of the buffer (see :func:`_wav_data_chunk`); any other chunk's size is + clipped to what is there. Raises ``ValueError`` if the buffer is not RIFF/WAVE. + + >>> raw = encode_wav_bytes([1, 2, 3], 42) + b'LIST' + struct.pack('>> [(chunk_id, size) for chunk_id, _, size in wav_chunks(raw)] + [(b'fmt ', 16), (b'data', 6), (b'LIST', 4)] + """ + _check_riff_wave(wav_bytes) + end, pos = len(wav_bytes), _RIFF_HEADER_SIZE + while pos + _CHUNK_HEADER_SIZE <= end: + chunk_id = bytes(wav_bytes[pos : pos + _CHUNK_ID_SIZE]) + (declared,) = struct.unpack( + " None: """Raise `ValueError` unless the buffer opens with a RIFF/WAVE preamble.""" if ( @@ -406,8 +457,13 @@ def _payload_end_of_overrunning_data(wav_bytes: bytes, contents: int) -> int: """ end = len(wav_bytes) start = contents + (contents % 2) - for candidate in range(start, end - _CHUNK_HEADER_SIZE + 1, 2): - if _parses_as_chunk_list_to_eof(wav_bytes, candidate): + # Only positions that read as a chunk id can start a chunk list, so a regex finds + # the candidates in C instead of trying every word of a long recording in Python. + for found in _CHUNK_ID_AHEAD.finditer( + wav_bytes, start, end - _CHUNK_HEADER_SIZE + 1 + ): + candidate = found.start() + if candidate % 2 == 0 and _parses_as_chunk_list_to_eof(wav_bytes, candidate): return candidate - contents return end - contents diff --git a/test_recode.py b/test_recode.py index ef688a7..7c917b7 100644 --- a/test_recode.py +++ b/test_recode.py @@ -789,3 +789,48 @@ def test_8_bit_encode_accepts_a_numpy_int8_waveform(): stereo = np.array([[-128, 1], [0, 127]], dtype=np.int8) raw = encode_wav_bytes(stereo, _SR, width_bytes=1, n_channels=2) assert decode_wav_bytes(raw) == ([(-128, 1), (0, 127)], _SR) + + +# --- wav_chunks: the one chunk walker (voxy reads its numpy arrays through it) --- + + +def _stream_wav(audio: bytes, *, declared=0xFFFFFFFF, after=b""): + head = b"RIFF" + struct.pack(" bytes: + return chunk_id + struct.pack(" Date: Mon, 9 Nov 2026 19:04:34 +0100 Subject: [PATCH 2/3] wav_chunks: known-id pass first, pad byte, WavChunk, decode_wav_bytes on the same walker (voxy#85) --- recode/audio.py | 65 +++++++++++++++++++++++++------------------------ test_recode.py | 27 +++++++++++++++++++- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/recode/audio.py b/recode/audio.py index f28e4d2..3f54648 100644 --- a/recode/audio.py +++ b/recode/audio.py @@ -42,7 +42,7 @@ import warnings import wave from io import BytesIO -from typing import Union +from typing import NamedTuple, Union from collections.abc import Iterable from numbers import Number from recode.base import mk_codec @@ -144,23 +144,11 @@ def decode_pcm_bytes(pcm_bytes: bytes, width: Width = 2, n_channels: int = 1): # Where a chunk id (four printable ASCII characters) starts, as a zero-width match. _CHUNK_ID_AHEAD = re.compile(rb"(?=[\x20-\x7e]{4})") # ... and the ids encoders append after the audio (metadata, cues, padding). -KNOWN_TRAILING_CHUNK_IDS = ( +_KNOWN_TRAILING_CHUNK_IDS = ( b"LIST", b"id3 ", b"ID3 ", b"fact", b"cue ", b"smpl", b"bext", b"JUNK", b"PAD ", b"iXML", b"_PMX", b"acid", b"inst", b"plst", ) # fmt: skip - - -def _known_chunk_id_positions(wav_bytes, start: int, stop: int): - """Where a known trailing chunk id occurs in ``wav_bytes[start:stop]``, in order - (``bytes.find`` scans in C, far faster than a regex alternation).""" - found = [] - for chunk_id in KNOWN_TRAILING_CHUNK_IDS: - at = wav_bytes.find(chunk_id, start, stop + _CHUNK_ID_SIZE) - while at != -1: - found.append(at) - at = wav_bytes.find(chunk_id, at + 1, stop + _CHUNK_ID_SIZE) - return sorted(found) - +_KNOWN_CHUNK_ID = re.compile(b"|".join(map(re.escape, _KNOWN_TRAILING_CHUNK_IDS))) class ShortWavData(UserWarning): @@ -351,17 +339,26 @@ def _wav_data_chunk(wav_bytes: bytes) -> tuple: >>> _wav_data_chunk(bytes(broken) + trailing) (44, 6) """ - contents, declared = _wav_chunk(wav_bytes, b"data") - if declared <= len(wav_bytes) - contents: - return contents, declared - return contents, _payload_end_of_overrunning_data(wav_bytes, contents) + for chunk in wav_chunks(wav_bytes): + if chunk.chunk_id == b"data": + return chunk.offset, chunk.size + _wav_chunk(wav_bytes, b"data") # no data chunk: raises the walk's own error + raise ValueError("Not WAV bytes: no `data` chunk found") + + +class WavChunk(NamedTuple): + """A RIFF chunk's id, and where its contents sit in the buffer.""" + + chunk_id: bytes + offset: int + size: int def wav_chunks(wav_bytes: bytes): - r"""``(chunk_id, offset, size)`` of every chunk of a RIFF/WAVE buffer, in order. + r""":class:`WavChunk` ``(chunk_id, offset, size)`` of every chunk of a RIFF/WAVE buffer, in order. The one chunk walker: :func:`decode_wav_bytes` and other readers (voxy's numpy - one) take their ``fmt `` and ``data`` from it. ``offset`` and ``size`` locate the + one) locate their ``data`` through it. ``offset`` and ``size`` locate the chunk's contents. A ``data`` chunk whose declared size is a streaming placeholder (``0``, or past the end of the buffer) ends where the chunks after it begin, or at the end of the buffer (see :func:`_wav_data_chunk`); any other chunk's size is @@ -383,7 +380,7 @@ def wav_chunks(wav_bytes: bytes): size = _payload_end_of_overrunning_data(wav_bytes, contents) else: size = min(declared, end - contents) - yield chunk_id, contents, size + yield WavChunk(chunk_id, contents, size) pos = contents + size + size % 2 @@ -456,20 +453,24 @@ def _payload_end_of_overrunning_data(wav_bytes: bytes, contents: int) -> int: file, where reading to the end is right. """ end = len(wav_bytes) - start = contents + (contents % 2) + start, last = contents + (contents % 2), end - _CHUNK_HEADER_SIZE + 1 # Only positions that read as a chunk id can start a chunk list, so a regex finds - # the candidates in C instead of trying every word of a long recording in Python. - for found in _CHUNK_ID_AHEAD.finditer( - wav_bytes, start, end - _CHUNK_HEADER_SIZE + 1 - ): - candidate = found.start() - if candidate % 2 == 0 and _parses_as_chunk_list_to_eof(wav_bytes, candidate): - return candidate - contents + # the candidates in C instead of trying every word in Python. The ids encoders + # really append come first: they are rare in audio, which makes that pass fast on + # a long recording. Any other printable id is the slower second pass (so an + # unknown chunk ahead of a known one is read as audio). + for pattern in (_KNOWN_CHUNK_ID, _CHUNK_ID_AHEAD): + for found in pattern.finditer(wav_bytes, start, last): + candidate = found.start() + if candidate % 2 == 0 and _parses_as_chunk_list_to_eof( + wav_bytes, candidate + ): + return candidate - contents return end - contents def _parses_as_chunk_list_to_eof(wav_bytes: bytes, pos: int) -> bool: - """Does the buffer from `pos` read as a chunk list that lands exactly on EOF?""" + """Does the buffer from `pos` read as a chunk list that lands on EOF (a final pad byte may be missing)?""" end = len(wav_bytes) while pos < end: if pos + _CHUNK_HEADER_SIZE > end: @@ -482,7 +483,7 @@ def _parses_as_chunk_list_to_eof(wav_bytes: bytes, pos: int) -> bool: " int: diff --git a/test_recode.py b/test_recode.py index 7c917b7..9dbaece 100644 --- a/test_recode.py +++ b/test_recode.py @@ -806,7 +806,9 @@ def _chunk(chunk_id: bytes, body: bytes) -> bytes: @pytest.mark.parametrize("declared", [0, 0xFFFFFFFF]) @pytest.mark.parametrize( - "audio", [bytes([1, 2]) * 500, bytes(1000)], ids=["noise", "digital-silence"] + "audio", + [bytes([1, 2]) * 500, bytes(1000), b"loud printable text " * 50], + ids=["noise", "digital-silence", "printable"], ) @pytest.mark.parametrize("chunk_id", [b"LIST", b"zzzz"], ids=["known", "unknown"]) def test_wav_chunks_ends_placeholder_data_before_trailing_chunks( @@ -834,3 +836,26 @@ def test_wav_chunks_clips_a_truncated_chunk_and_refuses_non_wav(): assert list(wav_chunks(raw))[-1][0::2] == (b"LIST", 2) with pytest.raises(ValueError, match="Not WAV"): list(wav_chunks(b"not a wav file at all")) + + +def test_wav_chunks_missing_final_pad_byte_and_buffer_types(): + from recode import wav_chunks + + trailer = b"LIST" + struct.pack(" Date: Mon, 9 Nov 2026 19:05:10 +0100 Subject: [PATCH 3/3] export WavChunk (voxy#85) --- recode/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recode/__init__.py b/recode/__init__.py index b48b577..c94143a 100644 --- a/recode/__init__.py +++ b/recode/__init__.py @@ -108,6 +108,7 @@ encode_wav_bytes, decode_wav_bytes, wav_chunks, + WavChunk, encode_wav_header_bytes, decode_wav_header_bytes, mk_pcm_audio_codec,