From 7d6b06310453d7fd3582cd28cbcf0c28e3d46f5d Mon Sep 17 00:00:00 2001 From: Tanishq Date: Thu, 12 Nov 2026 17:01:54 +0530 Subject: [PATCH 1/2] fix(websocket): pass client hotwords to AutoModel as words The websocket protocol sends hotwords as a JSON object string that maps each hotword to a weight, and the HTML5, Java, Android and C++ clients and the microphone mode of funasr_wss_client.py send that format. funasr_wss_server.py passed the string to AutoModel unchanged. AutoModel splits a hotword string on whitespace, so the JSON text became hotwords and the client hotwords had no effect. Keep the words of a JSON object and join them with spaces, as the C++ server does for NN hotwords. Other strings are passed on unchanged. --- runtime/python/websocket/funasr_wss_server.py | 21 ++- tests/test_websocket_server_hotwords.py | 121 ++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 tests/test_websocket_server_hotwords.py diff --git a/runtime/python/websocket/funasr_wss_server.py b/runtime/python/websocket/funasr_wss_server.py index 2a449e6423..7383177469 100644 --- a/runtime/python/websocket/funasr_wss_server.py +++ b/runtime/python/websocket/funasr_wss_server.py @@ -155,6 +155,25 @@ def _safe_int(v, default): return default +def _model_hotwords(hotwords): + """Turn protocol hotwords into the space-separated string AutoModel reads. + + Clients send hotwords as a JSON object string with weights, such as + '{"阿里巴巴": 20, "通义实验室": 30}' (runtime/docs/websocket_protocol.md). + AutoModel splits a hotword string on whitespace and takes no weights, so + keep only the words, as the C++ server does for NN hotwords. Any other + value is returned unchanged. + """ + if isinstance(hotwords, str): + try: + parsed = json.loads(hotwords) + except ValueError: + return hotwords + if isinstance(parsed, dict): + return " ".join(parsed) + return hotwords + + # ========= speaker db：加缓存，避免每段都读盘 ========= _SPEAKER_DB_CACHE = {} _SPEAKER_DB_CACHE_TS = 0.0 @@ -502,7 +521,7 @@ async def finish_input(send_end_ack): ] if "hotwords" in messagejson: - hotword_data = messagejson["hotwords"] + hotword_data = _model_hotwords(messagejson["hotwords"]) websocket.status_dict_asr["hotword"] = hotword_data websocket.status_dict_asr_online["hotword"] = hotword_data print(f"热词已更新: {hotword_data}") diff --git a/tests/test_websocket_server_hotwords.py b/tests/test_websocket_server_hotwords.py new file mode 100644 index 0000000000..36b0daeb39 --- /dev/null +++ b/tests/test_websocket_server_hotwords.py @@ -0,0 +1,121 @@ +"""Client hotwords in runtime/python/websocket/funasr_wss_server.py. + +The websocket protocol (runtime/docs/websocket_protocol.md) sends ``hotwords`` +as a JSON object string that maps each hotword to a weight, for example +'{"阿里巴巴":20,"通义实验室":30}'. The HTML5, Java, Android and C++ clients +and the microphone mode of funasr_wss_client.py all send this format. The +Python server passed the string to AutoModel unchanged, and AutoModel splits a +hotword string on whitespace, so the JSON text became ```` hotwords. +""" + +import asyncio +import importlib.util +import json +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SERVER_PATH = ROOT / "runtime/python/websocket/funasr_wss_server.py" + + +class FakeWebSocket: + def __init__(self, incoming): + self.incoming = iter(incoming) + self.sent = [] + self.path = None + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self.incoming) + except StopIteration: + raise StopAsyncIteration + + async def send(self, message): + self.sent.append(message) + + async def close(self): + pass + + +@pytest.fixture +def server(monkeypatch): + for name in ("websockets", "scipy", "torch"): + pytest.importorskip(name) + # The server builds its models with AutoModel when it is imported. + fake_funasr = types.ModuleType("funasr") + fake_funasr.AutoModel = lambda **kwargs: SimpleNamespace(**kwargs) + monkeypatch.setitem(sys.modules, "funasr", fake_funasr) + monkeypatch.setattr(sys, "argv", [str(SERVER_PATH)]) + spec = importlib.util.spec_from_file_location("funasr_wss_server_hotwords", SERVER_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + yield module + module.EXECUTOR.shutdown(wait=True) + + +def _offline_hotwords(server, hotwords): + """Send one offline request and return the hotword AutoModel.generate receives.""" + asr_calls = [] + + def asr_generate(input, **kwargs): + asr_calls.append(kwargs) + return [{"text": "阿里巴巴"}] + + def no_speaker_model(*args, **kwargs): + raise RuntimeError("no speaker model in this test") + + server.model_asr = SimpleNamespace(generate=asr_generate) + server.model_vad = SimpleNamespace(generate=lambda input, **kwargs: [{"value": []}]) + server.model_sv = SimpleNamespace(generate=no_speaker_model) + server.model_punc = None + config = { + "mode": "offline", + "chunk_size": [5, 10, 5], + "chunk_interval": 10, + "wav_name": "demo", + "is_speaking": True, + "hotwords": hotwords, + "itn": True, + } + websocket = FakeWebSocket( + [ + json.dumps(config, ensure_ascii=False), + b"\x01\x00" * 1600, + json.dumps({"is_speaking": False, "is_end": True}), + ] + ) + + asyncio.run(server.ws_serve(websocket)) + + assert json.loads(websocket.sent[-1])["is_final"] is True + assert len(asr_calls) == 1 + return asr_calls[0]["hotword"] + + +@pytest.mark.parametrize( + "hotwords, expected", + [ + # HTML5, Java, Android and C++ clients (compact JSON) + ('{"阿里巴巴":20,"通义实验室":30}', "阿里巴巴 通义实验室"), + # funasr_wss_client.py microphone mode (json.dumps) + ('{"阿里巴巴": 20, "通义实验室": 30}', "阿里巴巴 通义实验室"), + # an empty object: no hotwords + ("{}", ""), + ], +) +def test_protocol_hotwords_reach_the_model_as_words(server, hotwords, expected): + assert _offline_hotwords(server, hotwords) == expected + + +# A space-separated string is what AutoModel expects already, and +# funasr_wss_client.py sends "" when --hotword is not set. +@pytest.mark.parametrize("hotwords", ["阿里巴巴 通义实验室", ""]) +def test_plain_hotword_strings_are_unchanged(server, hotwords): + assert _offline_hotwords(server, hotwords) == hotwords From ff881eda188645d02e2b2223550fb426691cf4d7 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Fri, 13 Nov 2026 16:26:33 +0800 Subject: [PATCH 2/2] ci: integrate WebSocket protocol hotwords on current main Preserve original #3880 author ancestry and patch. Carry forward current main without rewriting history; include hotword regressions and real test dependencies. Signed-off-by: LauraGPT --- .../workflows/test-funasr-onnx-package.yml | 4 +- .github/workflows/test-python-websocket.yml | 55 ++++++++ README.md | 3 +- README_ja.md | 3 +- README_ko.md | 3 +- README_zh.md | 3 +- docs/repository_roles.md | 2 +- docs/repository_roles_zh.md | 2 +- funasr/utils/postprocess_utils.py | 11 +- funasr/version.txt | 2 +- .../libtorch/funasr_torch/paraformer_bin.py | 11 +- .../funasr_torch/utils/postprocess_utils.py | 11 +- runtime/python/onnxruntime/README.md | 10 +- .../onnxruntime/funasr_onnx/paraformer_bin.py | 15 ++- .../funasr_onnx/utils/postprocess_utils.py | 11 +- runtime/python/onnxruntime/setup.py | 2 +- runtime/python/websocket/funasr_wss_server.py | 6 +- tests/test_funasr_onnx_installed.py | 67 +++++++++- tests/test_funasr_onnx_release.py | 7 +- tests/test_release_version_contract.py | 15 ++- .../test_rich_transcription_event_segments.py | 4 +- tests/test_rich_transcription_postprocess.py | 21 +++ tests/test_runtime_eval_native.py | 123 ++++++++++++++++++ tests/test_websocket_server_asr_model.py | 64 +++++++++ .../product-site/content/legacy-manifest.json | 8 +- ...osted-deepgram-assemblyai-alternative.html | 3 +- ...hosted-openai-whisper-api-alternative.html | 3 +- ...osted-deepgram-assemblyai-alternative.html | 3 +- ...hosted-openai-whisper-api-alternative.html | 3 +- .../tests/test_blog_editorial_articles.py | 3 +- .../tests/test_http_blog_contracts.py | 17 ++- 31 files changed, 450 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/test-python-websocket.yml create mode 100644 tests/test_websocket_server_asr_model.py diff --git a/.github/workflows/test-funasr-onnx-package.yml b/.github/workflows/test-funasr-onnx-package.yml index 226421da66..18f0b4e542 100644 --- a/.github/workflows/test-funasr-onnx-package.yml +++ b/.github/workflows/test-funasr-onnx-package.yml @@ -86,7 +86,7 @@ jobs: import importlib.util from importlib.metadata import requires, version - assert version("funasr-onnx") == "0.4.10" + assert version("funasr-onnx") == "0.4.11" assert importlib.util.find_spec("torch") is None requirements = requires("funasr-onnx") or [] @@ -130,7 +130,7 @@ jobs: if: matrix.python-version == '3.12' uses: actions/upload-artifact@v7 with: - name: funasr-onnx-0.4.10 + name: funasr-onnx-0.4.11 path: dist/funasr-onnx/* if-no-files-found: error diff --git a/.github/workflows/test-python-websocket.yml b/.github/workflows/test-python-websocket.yml new file mode 100644 index 0000000000..d083808a3c --- /dev/null +++ b/.github/workflows/test-python-websocket.yml @@ -0,0 +1,55 @@ +name: Validate Python WebSocket + +on: + pull_request: + paths: + - "runtime/python/websocket/funasr_wss_server.py" + - "runtime/python/websocket/funasr_wss_client.py" + - "tests/test_websocket_server_asr_model.py" + - "tests/test_websocket_server_hotwords.py" + - "tests/test_websocket_file_finalization.py" + - ".github/workflows/test-python-websocket.yml" + push: + branches: + - main + paths: + - "runtime/python/websocket/funasr_wss_server.py" + - "runtime/python/websocket/funasr_wss_client.py" + - "tests/test_websocket_server_asr_model.py" + - "tests/test_websocket_server_hotwords.py" + - "tests/test_websocket_file_finalization.py" + - ".github/workflows/test-python-websocket.yml" + +permissions: + contents: read + +jobs: + websocket: + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install isolated protocol test dependencies + run: | + python -m pip install pytest numpy scipy websockets + python -m pip install torch --index-url https://download.pytorch.org/whl/cpu + - name: Run constructor, hotword and file-finalization regressions + run: >- + python -m pytest -q + tests/test_websocket_server_asr_model.py + tests/test_websocket_server_hotwords.py + tests/test_websocket_file_finalization.py + --junitxml=websocket-test-results.xml + - uses: actions/upload-artifact@v4 + if: always() + with: + name: websocket-tests-py${{ matrix.python-version }} + path: websocket-test-results.xml + if-no-files-found: error diff --git a/README.md b/README.md index 108711e432..53c94fa55b 100644 --- a/README.md +++ b/README.md @@ -364,7 +364,8 @@ recordings with the same evaluation scope. ## What's new - **MOSS-Transcribe-Diarize** brings long-form ASR, timestamps, and anonymous speaker labels to FunASR services, Docker, Kubernetes, vLLM/SGLang workflows, and FunClip. [Deploy MOSS ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.25** restores Python 3.12 boolean argument parsing without `distutils` and preserves normalized error messages; removes synthetic leading spaces from Latin sentence timestamps while preserving boundaries and raw text. [Fix #3863](https://github.com/modelscope/FunASR/pull/3863), [fix #3866](https://github.com/modelscope/FunASR/pull/3866). Install with `python -m pip install -U "funasr==1.4.25"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) +- **FunASR 1.4.26** preserves word spaces between English/Korean VAD segments and returns plain text for HTTP `response_format=text`. [#3874](https://github.com/modelscope/FunASR/pull/3874), [#3883](https://github.com/modelscope/FunASR/pull/3883). `python -m pip install -U "funasr==1.4.26"`. [Release / 验证范围](https://github.com/modelscope/FunASR/releases/tag/v1.4.26) + **FunASR 1.4.25** restores Python 3.12 boolean argument parsing without `distutils` and preserves normalized error messages; removes synthetic leading spaces from Latin sentence timestamps while preserving boundaries and raw text. [Fix #3863](https://github.com/modelscope/FunASR/pull/3863), [fix #3866](https://github.com/modelscope/FunASR/pull/3866). Install with `python -m pip install -U "funasr==1.4.25"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) **FunASR 1.4.24** ignores empty non-linguistic symbols to prevent tokenizer loops, fixes padded variance normalization, and prevents CIF crashes in mixed batches containing zero-fire samples. [Fix #3852](https://github.com/modelscope/FunASR/pull/3852), [fix #3853](https://github.com/modelscope/FunASR/pull/3853), [fix #3854](https://github.com/modelscope/FunASR/pull/3854). A checkpoint trained with both `norm_means=True` and `norm_vars=True` needs recognition-quality reevaluation because normalized features change; `norm_vars=False` remains the default. Install with `python -m pip install -U "funasr==1.4.24"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.24) **FunASR 1.4.23** fixes the online frontend cursor for `lfr_m=lfr_n=1` and redundant empty final offline VAD chunks. With a supplied `token_list`, SentencePiece `ids2tokens` returns model pieces; use `tokens2text` for text. Without `token_list`, legacy text decoding is unchanged. [Fix #3849](https://github.com/modelscope/FunASR/pull/3849), [fix #3783](https://github.com/modelscope/FunASR/pull/3783). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) **FunASR 1.4.22** honors per-call `merge_length_s` and evaluates dynamic VAD silence thresholds per frame using the current segment duration, preserving the fixed-threshold opt-out. [Fix #3841](https://github.com/modelscope/FunASR/pull/3841), [fix #3833](https://github.com/modelscope/FunASR/pull/3833). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) diff --git a/README_ja.md b/README_ja.md index 64f634fe64..883faf2c03 100644 --- a/README_ja.md +++ b/README_ja.md @@ -112,7 +112,8 @@ CER/WER をそろえて比較してください。オフラインのスループ ## 最新情報 - **MOSS-Transcribe-Diarize** を FunASR service、Docker、Kubernetes、vLLM/SGLang workflow、FunClip に統合し、長時間 ASR、timestamp、匿名 speaker label を一度に処理できます。[MOSS をデプロイ ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.25** は Python 3.12 の真偽値引数を `distutils` なしで解析し、正規化されたエラーメッセージを維持します。ラテン文字で始まる文のタイムスタンプ出力から余分な先頭空白を除き、時刻境界と原文を保持します。[修正 #3863](https://github.com/modelscope/FunASR/pull/3863)、[修正 #3866](https://github.com/modelscope/FunASR/pull/3866)。`python -m pip install -U "funasr==1.4.25"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) +- **FunASR 1.4.26** は英語・韓国語の VAD セグメント間の単語区切りを保持し、HTTP `response_format=text` でプレーンテキストを返します。 [#3874](https://github.com/modelscope/FunASR/pull/3874), [#3883](https://github.com/modelscope/FunASR/pull/3883). `python -m pip install -U "funasr==1.4.26"`. [Release / 验证范围](https://github.com/modelscope/FunASR/releases/tag/v1.4.26) + **FunASR 1.4.25** は Python 3.12 の真偽値引数を `distutils` なしで解析し、正規化されたエラーメッセージを維持します。ラテン文字で始まる文のタイムスタンプ出力から余分な先頭空白を除き、時刻境界と原文を保持します。[修正 #3863](https://github.com/modelscope/FunASR/pull/3863)、[修正 #3866](https://github.com/modelscope/FunASR/pull/3866)。`python -m pip install -U "funasr==1.4.25"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) **FunASR 1.4.24** は空の非言語記号によるトークナイザーの無限ループ、パディングを含む分散正規化、発火回数ゼロのサンプルを含む混合バッチでの CIF クラッシュを修正します。[修正 #3852](https://github.com/modelscope/FunASR/pull/3852)、[修正 #3853](https://github.com/modelscope/FunASR/pull/3853)、[修正 #3854](https://github.com/modelscope/FunASR/pull/3854)。`norm_means=True` と `norm_vars=True` の両方で学習した checkpoint は正規化特徴が変わるため、認識品質の再評価が必要です。既定値は引き続き `norm_vars=False` です。`python -m pip install -U "funasr==1.4.24"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.24) **FunASR 1.4.23** は `lfr_m=lfr_n=1` のオンライン特徴抽出のフレーム位置と、オフライン VAD の余分な空の最終チャンクを修正します。`token_list` 指定時の SentencePiece `ids2tokens` はモデルのピースを返し、テキストへの変換には `tokens2text` を使います。未指定時の従来のテキスト復号は維持します。[修正 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修正 #3783](https://github.com/modelscope/FunASR/pull/3783)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) **FunASR 1.4.22** は呼び出しごとの `merge_length_s` を反映し、現在の発話区間の長さに基づいて動的 VAD 無音しきい値をフレーム単位で更新します。固定しきい値モードも維持します。[修正 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修正 #3833](https://github.com/modelscope/FunASR/pull/3833)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) diff --git a/README_ko.md b/README_ko.md index caa4938595..43e51b4dad 100644 --- a/README_ko.md +++ b/README_ko.md @@ -111,7 +111,8 @@ checkpoint/revision, 오디오 집합, 하드웨어, 배치, 워밍업, 측정 ## 최신 소식 - **MOSS-Transcribe-Diarize**를 FunASR service, Docker, Kubernetes, vLLM/SGLang workflow, FunClip에 통합해 긴 오디오 ASR, timestamp, 익명 speaker label을 한 번에 처리합니다. [MOSS 배포 ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.25**는 Python 3.12에서 `distutils` 없이 불리언 인수를 처리하고 정규화된 오류 메시지를 유지합니다. 라틴 문자로 시작하는 문장의 타임스탬프 출력에서 불필요한 앞 공백을 제거하며 시간 경계와 원문은 보존합니다. [수정 #3863](https://github.com/modelscope/FunASR/pull/3863), [수정 #3866](https://github.com/modelscope/FunASR/pull/3866). `python -m pip install -U "funasr==1.4.25"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) +- **FunASR 1.4.26** 는 영어·한국어 VAD 구간 사이의 단어 공백을 보존하고 HTTP `response_format=text` 요청에 일반 텍스트를 반환합니다. [#3874](https://github.com/modelscope/FunASR/pull/3874), [#3883](https://github.com/modelscope/FunASR/pull/3883). `python -m pip install -U "funasr==1.4.26"`. [Release / 验证范围](https://github.com/modelscope/FunASR/releases/tag/v1.4.26) + **FunASR 1.4.25**는 Python 3.12에서 `distutils` 없이 불리언 인수를 처리하고 정규화된 오류 메시지를 유지합니다. 라틴 문자로 시작하는 문장의 타임스탬프 출력에서 불필요한 앞 공백을 제거하며 시간 경계와 원문은 보존합니다. [수정 #3863](https://github.com/modelscope/FunASR/pull/3863), [수정 #3866](https://github.com/modelscope/FunASR/pull/3866). `python -m pip install -U "funasr==1.4.25"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) **FunASR 1.4.24**는 빈 비언어 기호로 인한 토크나이저 무한 루프, 패딩이 있는 입력의 분산 정규화 오류, 발화 횟수가 0인 샘플이 섞인 배치의 CIF 충돌을 수정합니다. [수정 #3852](https://github.com/modelscope/FunASR/pull/3852), [수정 #3853](https://github.com/modelscope/FunASR/pull/3853), [수정 #3854](https://github.com/modelscope/FunASR/pull/3854). `norm_means=True`와 `norm_vars=True`를 모두 사용해 학습한 checkpoint는 정규화 특징이 달라지므로 인식 품질을 다시 평가해야 합니다. 기본값은 계속 `norm_vars=False`입니다. `python -m pip install -U "funasr==1.4.24"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.24) **FunASR 1.4.23**은 `lfr_m=lfr_n=1` 온라인 프런트엔드의 프레임 위치와 오프라인 VAD의 불필요한 빈 마지막 청크를 수정합니다. `token_list`를 지정하면 SentencePiece `ids2tokens`는 모델 토큰을 반환하며, 텍스트 변환에는 `tokens2text`를 사용합니다. 미지정 시 기존 텍스트 디코딩은 유지됩니다. [수정 #3849](https://github.com/modelscope/FunASR/pull/3849), [수정 #3783](https://github.com/modelscope/FunASR/pull/3783). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) **FunASR 1.4.22**는 호출별 `merge_length_s`를 반영하고 현재 발화 구간 길이에 따라 동적 VAD 무음 임계값을 프레임마다 갱신합니다. 고정 임계값 모드도 유지합니다. [수정 #3841](https://github.com/modelscope/FunASR/pull/3841), [수정 #3833](https://github.com/modelscope/FunASR/pull/3833). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) diff --git a/README_zh.md b/README_zh.md index ec13d2f340..efa77d7ca4 100644 --- a/README_zh.md +++ b/README_zh.md @@ -157,7 +157,8 @@ checkpoint/revision、音频集、硬件、批量大小、预热、计时范围 ## 最新动态 - **MOSS-Transcribe-Diarize** 已接入 FunASR 服务、Docker、Kubernetes、vLLM/SGLang 工作流和 FunClip，一次完成长音频转写、时间戳与匿名说话人标注。[部署 MOSS ->](./docs/moss_transcribe_diarize_zh.md) -- **FunASR 1.4.25** 修复 Python 3.12 下不依赖 `distutils` 的布尔参数解析并保留归一化错误信息；移除拉丁文字句子时间戳中的多余前导空格，保留时间边界和原始文本。[修复 #3863](https://github.com/modelscope/FunASR/pull/3863)、[修复 #3866](https://github.com/modelscope/FunASR/pull/3866)。安装：`python -m pip install -U "funasr==1.4.25"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) +- **FunASR 1.4.26** 保留英语、韩语 VAD 分段间的词间空格，并让 HTTP `response_format=text` 返回纯文本。 [#3874](https://github.com/modelscope/FunASR/pull/3874), [#3883](https://github.com/modelscope/FunASR/pull/3883). `python -m pip install -U "funasr==1.4.26"`. [Release / 验证范围](https://github.com/modelscope/FunASR/releases/tag/v1.4.26) + **FunASR 1.4.25** 修复 Python 3.12 下不依赖 `distutils` 的布尔参数解析并保留归一化错误信息；移除拉丁文字句子时间戳中的多余前导空格，保留时间边界和原始文本。[修复 #3863](https://github.com/modelscope/FunASR/pull/3863)、[修复 #3866](https://github.com/modelscope/FunASR/pull/3866)。安装：`python -m pip install -U "funasr==1.4.25"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.25) **FunASR 1.4.24** 忽略空的非语言符号以避免分词死循环，修复带填充输入的方差归一化，并避免包含零触发样本的混合批次导致 CIF 崩溃。[修复 #3852](https://github.com/modelscope/FunASR/pull/3852)、[修复 #3853](https://github.com/modelscope/FunASR/pull/3853)、[修复 #3854](https://github.com/modelscope/FunASR/pull/3854)。同时启用 `norm_means=True` 和 `norm_vars=True` 训练的 checkpoint 会收到不同的归一化特征，需要重新评估识别质量；默认仍为 `norm_vars=False`。安装：`python -m pip install -U "funasr==1.4.24"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.24) **FunASR 1.4.23** 修复 `lfr_m=lfr_n=1` 在线前端的帧游标和离线 VAD 多余的空尾块。提供 `token_list` 时，SentencePiece 的 `ids2tokens` 返回模型词元，请用 `tokens2text` 转为文本；未提供 `token_list` 时保持旧有文本解码行为。[修复 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修复 #3783](https://github.com/modelscope/FunASR/pull/3783)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) **FunASR 1.4.22** 支持逐调用覆盖 `merge_length_s`，并根据当前语音段时长逐帧更新动态 VAD 静音阈值，同时保留固定阈值模式。[修复 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修复 #3833](https://github.com/modelscope/FunASR/pull/3833)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) diff --git a/docs/repository_roles.md b/docs/repository_roles.md index bff0e53a91..30178f163b 100644 --- a/docs/repository_roles.md +++ b/docs/repository_roles.md @@ -5,7 +5,7 @@ This document explains the responsibility boundaries, user entry points, and iss > **Directional roadmap, not a release promise.** > This document records shipped capabilities and active work, but does not commit to > future version numbers or dates. The current Python release is -> [`funasr==1.4.25`](https://github.com/modelscope/FunASR/releases/tag/v1.4.25). +> [`funasr==1.4.26`](https://github.com/modelscope/FunASR/releases/tag/v1.4.26). > Any future breaking release still requires a maintainer-approved milestone and > migration plan. diff --git a/docs/repository_roles_zh.md b/docs/repository_roles_zh.md index fd979acedf..013f0669e0 100644 --- a/docs/repository_roles_zh.md +++ b/docs/repository_roles_zh.md @@ -4,7 +4,7 @@ > **方向性路线图，不是版本承诺。** > 本文档记录已交付能力与正在推进的工作，但不承诺未来版本号或日期。当前 Python -> 版本是 [`funasr==1.4.25`](https://github.com/modelscope/FunASR/releases/tag/v1.4.25)。 +> 版本是 [`funasr==1.4.26`](https://github.com/modelscope/FunASR/releases/tag/v1.4.26)。 > 任何未来的 breaking release 仍需 maintainer 确认 milestone 与迁移方案。 --- diff --git a/funasr/utils/postprocess_utils.py b/funasr/utils/postprocess_utils.py index 6a3cd2336c..cd99e1fbbe 100644 --- a/funasr/utils/postprocess_utils.py +++ b/funasr/utils/postprocess_utils.py @@ -442,6 +442,11 @@ def format_str_v2(s): return s.strip() +def _is_spaced_char(ch): + """Latin and Hangul text separates words with spaces; CJK text does not.""" + return (ch.isascii() and not ch.isspace()) or "\uac00" <= ch <= "\ud7a3" + + def rich_transcription_postprocess(s): """Rich transcription postprocess. @@ -481,6 +486,10 @@ def get_event(s): cur_ent_event = get_event(s_list[i]) if get_emo(s_list[i]) != None and get_emo(s_list[i]) == get_emo(new_s): new_s = new_s[:-1] - new_s += s_list[i].strip().lstrip() + segment = s_list[i].strip() + # Each VAD segment is its own <|lang|> block; keep the word boundary between them. + if segment and _is_spaced_char(new_s[-1]) and _is_spaced_char(segment[0]): + new_s += " " + new_s += segment new_s = new_s.replace("The.", " ") return new_s.strip() diff --git a/funasr/version.txt b/funasr/version.txt index df3487bee1..24d07aa70a 100644 --- a/funasr/version.txt +++ b/funasr/version.txt @@ -1 +1 @@ -1.4.25 +1.4.26 diff --git a/runtime/python/libtorch/funasr_torch/paraformer_bin.py b/runtime/python/libtorch/funasr_torch/paraformer_bin.py index dc9d2a5966..15bab50753 100644 --- a/runtime/python/libtorch/funasr_torch/paraformer_bin.py +++ b/runtime/python/libtorch/funasr_torch/paraformer_bin.py @@ -120,10 +120,10 @@ def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> pred = sentence_postprocess(pred) asr_res.append({"preds": pred}) else: - for pred, us_peaks_ in zip(preds, us_peaks): + for pred, us_peaks_, feat_len in zip(preds, us_peaks, feats_len): raw_tokens = pred timestamp, timestamp_raw = time_stamp_lfr6_onnx( - us_peaks_, copy.copy(raw_tokens) + us_peaks_[: feat_len * 3], copy.copy(raw_tokens) ) text_proc, timestamp_proc, _ = sentence_postprocess( raw_tokens, timestamp_raw @@ -365,7 +365,10 @@ def __call__( return asr_res def proc_hotword(self, hotwords): - hotwords = hotwords.split(" ") + # Like the C++ runtime, skip empty hotwords and keep the first max_hotword_len + # characters of longer ones: the exported embedding model takes that many per hotword. + max_hotword_len = 10 + hotwords = [hotword[:max_hotword_len] for hotword in hotwords.split()] hotwords_length = [len(i) - 1 for i in hotwords] hotwords_length.append(0) hotwords_length = np.array(hotwords_length) @@ -385,7 +388,7 @@ def word_map(word): hotword_int = [word_map(i) for i in hotwords] hotword_int.append(np.array([1])) - hotwords = pad_list(hotword_int, pad_value=0, max_len=10) + hotwords = pad_list(hotword_int, pad_value=0, max_len=max_hotword_len) return torch.tensor(hotwords), hotwords_length def bb_infer( diff --git a/runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py b/runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py index 34180841cc..d5b7a6ffb3 100644 --- a/runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py +++ b/runtime/python/libtorch/funasr_torch/utils/postprocess_utils.py @@ -347,6 +347,11 @@ def format_str_v2(s): return s.strip() +def _is_spaced_char(ch): + """Latin and Hangul text separates words with spaces; CJK text does not.""" + return (ch.isascii() and not ch.isspace()) or "\uac00" <= ch <= "\ud7a3" + + def rich_transcription_postprocess(s): def get_emo(s): return s[-1] if s[-1] in emo_set else None @@ -371,6 +376,10 @@ def get_event(s): cur_ent_event = get_event(s_list[i]) if get_emo(s_list[i]) != None and get_emo(s_list[i]) == get_emo(new_s): new_s = new_s[:-1] - new_s += s_list[i].strip().lstrip() + segment = s_list[i].strip() + # Each VAD segment is its own <|lang|> block; keep the word boundary between them. + if segment and _is_spaced_char(new_s[-1]) and _is_spaced_char(segment[0]): + new_s += " " + new_s += segment new_s = new_s.replace("The.", " ") return new_s.strip() diff --git a/runtime/python/onnxruntime/README.md b/runtime/python/onnxruntime/README.md index e3d18c6d43..4982cc4504 100644 --- a/runtime/python/onnxruntime/README.md +++ b/runtime/python/onnxruntime/README.md @@ -7,6 +7,12 @@ does not update the installed ONNX wrappers. Check their versions separately with `python -m pip show funasr-onnx funasr`; use the source installation below when a required wrapper fix is newer than the published PyPI package. +Version 0.4.11 keeps timestamps independent of padded samples in a batch +(#3871), bounds Contextual/Seaco hotwords to the exported ten-character +embedding input and skips whitespace-only entries (#3882), and preserves +terminal abbreviations and their timestamps (#3869). These are wrapper and +postprocessing fixes, not model-weight updates or ASR accuracy claims. + Version 0.4.10 restores direct librosa decoding for non-WAV audio in offline and streaming Paraformer, SenseVoice and offline VAD (#3865), without creating a WAV sidecar or treating every non-WAV input as MP3. Codec support still depends on the @@ -45,9 +51,9 @@ and postprocessing fixes, not a model-weight update or an ASR accuracy claim. install from pip ```shell -pip install -U "funasr-onnx>=0.4.10" +pip install -U "funasr-onnx>=0.4.11" # For the users in China, you could install with the command: -# pip install -U "funasr-onnx>=0.4.10" -i https://mirror.sjtu.edu.cn/pypi/web/simple +# pip install -U "funasr-onnx>=0.4.11" -i https://mirror.sjtu.edu.cn/pypi/web/simple # If you want to export .onnx file, you should install modelscope and funasr pip install -U modelscope funasr # For the users in China, you could install with the command: diff --git a/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py b/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py index 517252cd85..fff3f301d3 100644 --- a/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py @@ -131,10 +131,10 @@ def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> pred = sentence_postprocess(pred) asr_res.append({"preds": pred}) else: - for pred, us_peaks_ in zip(preds, us_peaks): + for pred, us_peaks_, feat_len in zip(preds, us_peaks, feats_len): raw_tokens = pred timestamp, timestamp_raw = time_stamp_lfr6_onnx( - us_peaks_, copy.copy(raw_tokens) + us_peaks_[: feat_len * 3], copy.copy(raw_tokens) ) text_proc, timestamp_proc, _ = sentence_postprocess( raw_tokens, timestamp_raw @@ -384,10 +384,10 @@ def __call__( pred = sentence_postprocess(pred) asr_res.append({"preds": pred}) else: - for pred, us_peaks_ in zip(preds, us_peaks): + for pred, us_peaks_, feat_len in zip(preds, us_peaks, feats_len): raw_tokens = pred timestamp, timestamp_raw = time_stamp_lfr6_onnx( - us_peaks_, copy.copy(raw_tokens) + us_peaks_[: feat_len * 3], copy.copy(raw_tokens) ) text_proc, timestamp_proc, _ = sentence_postprocess( raw_tokens, timestamp_raw @@ -407,7 +407,10 @@ def __call__( return asr_res def proc_hotword(self, hotwords): - hotwords = hotwords.split(" ") + # Like the C++ runtime, skip empty hotwords and keep the first max_hotword_len + # characters of longer ones: the exported embedding model takes that many per hotword. + max_hotword_len = 10 + hotwords = [hotword[:max_hotword_len] for hotword in hotwords.split()] hotwords_length = [len(i) - 1 for i in hotwords] hotwords_length.append(0) hotwords_length = np.array(hotwords_length) @@ -428,7 +431,7 @@ def word_map(word): hotword_int = [word_map(i) for i in hotwords] hotword_int.append(np.array([1])) - hotwords = pad_list(hotword_int, pad_value=0, max_len=10) + hotwords = pad_list(hotword_int, pad_value=0, max_len=max_hotword_len) return hotwords, hotwords_length diff --git a/runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py b/runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py index 3a62afee66..82c334f855 100644 --- a/runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py +++ b/runtime/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py @@ -401,6 +401,11 @@ def format_str_v2(s): return s.strip() +def _is_spaced_char(ch): + """Latin and Hangul text separates words with spaces; CJK text does not.""" + return (ch.isascii() and not ch.isspace()) or "\uac00" <= ch <= "\ud7a3" + + def rich_transcription_postprocess(s): def get_emo(s): return s[-1] if s[-1] in emo_set else None @@ -425,6 +430,10 @@ def get_event(s): cur_ent_event = get_event(s_list[i]) if get_emo(s_list[i]) != None and get_emo(s_list[i]) == get_emo(new_s): new_s = new_s[:-1] - new_s += s_list[i].strip().lstrip() + segment = s_list[i].strip() + # Each VAD segment is its own <|lang|> block; keep the word boundary between them. + if segment and _is_spaced_char(new_s[-1]) and _is_spaced_char(segment[0]): + new_s += " " + new_s += segment new_s = new_s.replace("The.", " ") return new_s.strip() diff --git a/runtime/python/onnxruntime/setup.py b/runtime/python/onnxruntime/setup.py index 40290072f7..22791fa140 100644 --- a/runtime/python/onnxruntime/setup.py +++ b/runtime/python/onnxruntime/setup.py @@ -13,7 +13,7 @@ def get_readme(): MODULE_NAME = "funasr_onnx" -VERSION_NUM = "0.4.10" +VERSION_NUM = "0.4.11" setuptools.setup( name=MODULE_NAME, diff --git a/runtime/python/websocket/funasr_wss_server.py b/runtime/python/websocket/funasr_wss_server.py index 7383177469..8dd4ddb22a 100644 --- a/runtime/python/websocket/funasr_wss_server.py +++ b/runtime/python/websocket/funasr_wss_server.py @@ -45,7 +45,7 @@ def to_python(obj): parser.add_argument( "--asr_model", type=str, - default="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404", + default="paraformer-zh", help="model from modelscope", ) parser.add_argument("--asr_model_revision", type=str, default="v2.0.4", help="") @@ -254,8 +254,8 @@ def save_offline_wav_segment_sync(websocket, audio_bytes: bytes, reason: str = " # ====== 离线 ASR ====== model_asr = AutoModel( - model="paraformer-zh", - model_revision="v2.0.4", + model=args.asr_model, + model_revision=args.asr_model_revision, ngpu=args.ngpu, ncpu=args.ncpu, device=args.device, diff --git a/tests/test_funasr_onnx_installed.py b/tests/test_funasr_onnx_installed.py index b6a74c78ef..1618704544 100644 --- a/tests/test_funasr_onnx_installed.py +++ b/tests/test_funasr_onnx_installed.py @@ -41,6 +41,71 @@ class InstalledOnnxPackageTest(unittest.TestCase): LOADERS = (funasr_onnx.Paraformer, ParaformerOnline, funasr_onnx.SenseVoiceSmall, funasr_onnx.Fsmn_vad) + def test_long_and_empty_hotwords_stay_within_embedding_bounds(self): + for cls in (funasr_onnx.ContextualParaformer, funasr_onnx.SeacoParaformer): + with self.subTest(wrapper=cls.__name__): + model = cls.__new__(cls) + model.vocab = {'a': 3, 'b': 4} + matrix, positions = model.proc_hotword(' ab abababababab ') + self.assertEqual(matrix.tolist(), [ + [3, 4, 0, 0, 0, 0, 0, 0, 0, 0], + [3, 4, 3, 4, 3, 4, 3, 4, 3, 4], + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ]) + self.assertEqual(positions.tolist(), [1, 9, 0]) + for value in ('', ' \t\n'): + matrix, positions = model.proc_hotword(value) + self.assertEqual(matrix.tolist(), [[1, 0, 0, 0, 0, 0, 0, 0, 0, 0]]) + self.assertEqual(positions.tolist(), [0]) + + def test_batched_timestamps_use_each_input_length(self): + def features(lengths): + return np.zeros((len(lengths), max(lengths), 4), dtype=np.float32), np.array(lengths) + + def acoustic_output(feats, lengths): + peaks = np.zeros((len(lengths), feats.shape[1] * 3), dtype=np.float32) + for row, length in enumerate(lengths): + peaks[row, np.linspace(5, length * 3 - 6, 3).round().astype(int)] = 1 + return None, None, np.zeros_like(peaks), peaks + + # Acoustic outputs are synthetic. The installed batching, timestamp and + # postprocessing code must keep the shorter sample independent of padding. + for cls in (funasr_onnx.Paraformer, funasr_onnx.ContextualParaformer, + funasr_onnx.SeacoParaformer): + with self.subTest(wrapper=cls.__name__): + model = cls.__new__(cls) + model.batch_size = 2 + model.plot_timestamp_to = '' + model.language = 'zh' + model.frontend = SimpleNamespace(opts=SimpleNamespace( + frame_opts=SimpleNamespace(samp_freq=16000))) + model.load_data = lambda values, fs: values + model.extract_feat = features + model.decode = lambda scores, lens: [['\u4f60', '\u597d'] for _ in range(model._rows)] + + def infer(feats, lengths, *unused): + model._rows = len(lengths) + return acoustic_output(feats, lengths) + + model.infer = infer + model.bb_infer = infer + model.vocab = {'a': 3} + model.eb_infer = lambda hotwords, lengths: [np.zeros((10, len(hotwords), 4), dtype=np.float32)] + args = () if cls is funasr_onnx.Paraformer else ('a',) + single = [model([length], *args)[0] for length in [17, 50]] + self.assertEqual([x['timestamp'] for x in single], [ + [[70, 470], [470, 945]], [[70, 670], [1450, 2925]], + ]) + self.assertEqual(model([17, 50], *args), single) + + def test_terminal_abbreviation_preserves_text_and_timestamps(self): + from funasr_onnx.utils.postprocess_utils import abbr_dispose + + self.assertEqual(abbr_dispose(['a', ' ', 'b']), ['AB']) + self.assertEqual(abbr_dispose(['a', ' ', 'b'], [[0, 100], [100, 200]]), + (['AB'], [[0, 200]])) + self.assertEqual(abbr_dispose(['a', ' ', 'b', ' ', 'c']), ['ABC']) + def test_lossless_non_wav_without_sidecar(self): samples = (np.arange(4800, dtype=np.int16) - 2400).astype(np.float32) / 32768 with tempfile.TemporaryDirectory() as directory: @@ -91,7 +156,7 @@ def test_invalid_audio_remains_visible_without_sidecar(self): def test_installed_distribution_version_and_origin(self): distribution = importlib.metadata.distribution('funasr-onnx') - self.assertEqual(distribution.version, '0.4.10') + self.assertEqual(distribution.version, '0.4.11') self.assertEqual( Path(funasr_onnx.__file__).resolve().parent, Path(distribution.locate_file('funasr_onnx')).resolve(), diff --git a/tests/test_funasr_onnx_release.py b/tests/test_funasr_onnx_release.py index cb57f6be51..1e73caf87e 100644 --- a/tests/test_funasr_onnx_release.py +++ b/tests/test_funasr_onnx_release.py @@ -7,7 +7,7 @@ ROOT = Path(__file__).resolve().parents[1] PACKAGE_ROOT = ROOT / "runtime" / "python" / "onnxruntime" SETUP_PATH = PACKAGE_ROOT / "setup.py" -EXPECTED_VERSION = "0.4.10" +EXPECTED_VERSION = "0.4.11" REQUIREMENT_NAME_PATTERN = re.compile(r"^\s*([A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?)") @@ -44,7 +44,7 @@ def normalized_requirement_name(requirement): class FunASROnnxReleaseContractTest(unittest.TestCase): - def test_release_version_is_0_4_10(self): + def test_release_version_is_0_4_11(self): self.assertEqual(assigned_literal(read_setup_tree(), "VERSION_NUM"), EXPECTED_VERSION) def test_standalone_license_preserves_repository_notice(self): @@ -83,6 +83,9 @@ def test_readme_names_the_standalone_fix_release(self): self.assertIn("#3847", readme) self.assertIn("empty or sub-frame", readme) self.assertIn("preserves the streaming cache", readme) + self.assertIn("#3869", readme) + self.assertIn("#3871", readme) + self.assertIn("#3882", readme) def test_runtime_dependencies_keep_onnx_install_torch_free(self): requirements = setup_keyword_literal(read_setup_tree(), "install_requires") diff --git a/tests/test_release_version_contract.py b/tests/test_release_version_contract.py index d2b076db28..b3bf951823 100644 --- a/tests/test_release_version_contract.py +++ b/tests/test_release_version_contract.py @@ -4,7 +4,7 @@ ROOT = Path(__file__).resolve().parents[1] -RELEASE_VERSION = "1.4.25" +RELEASE_VERSION = "1.4.26" README_FILES = ("README.md", "README_zh.md", "README_ja.md", "README_ko.md") REPOSITORY_ROLE_FILES = ("docs/repository_roles.md", "docs/repository_roles_zh.md") @pytest.mark.parametrize("name", README_FILES) @@ -23,7 +23,7 @@ def test_release_summary_covers_tokenizer_normalizer_and_cif(name): @pytest.mark.parametrize("name", README_FILES) def test_release_summary_covers_parser_and_timestamp_fixes(name): summaries = [line for line in (ROOT / name).read_text(encoding="utf-8").splitlines() - if f"**FunASR {RELEASE_VERSION}**" in line] + if "**FunASR 1.4.25**" in line] assert len(summaries) == 1 assert "Python 3.12" in summaries[0] assert "distutils" in summaries[0] @@ -31,6 +31,17 @@ def test_release_summary_covers_parser_and_timestamp_fixes(name): assert f"https://github.com/modelscope/FunASR/pull/{number}" in summaries[0] +@pytest.mark.parametrize("name", README_FILES) +def test_release_summary_covers_vad_spacing_and_plain_text(name): + summaries = [line for line in (ROOT / name).read_text(encoding="utf-8").splitlines() + if f"**FunASR {RELEASE_VERSION}**" in line] + assert len(summaries) == 1 + assert "VAD" in summaries[0] + assert "response_format=text" in summaries[0] + for number in (3874, 3883): + assert f"https://github.com/modelscope/FunASR/pull/{number}" in summaries[0] + + diff --git a/tests/test_rich_transcription_event_segments.py b/tests/test_rich_transcription_event_segments.py index ed756abfd8..a565b5c3cd 100644 --- a/tests/test_rich_transcription_event_segments.py +++ b/tests/test_rich_transcription_event_segments.py @@ -34,7 +34,9 @@ def test_repeated_event_only_segments_keep_following_text(postprocess, event, fo result = postprocess.rich_transcription_postprocess(text) - assert result == postprocess.event_dict[event] + "hello" + following_text + assert result == postprocess.event_dict[event] + " ".join( + filter(None, ["hello", following_text]) + ) @pytest.mark.parametrize("event", ["<|Cough|>", "<|Sneeze|>", "<|Applause|>"]) diff --git a/tests/test_rich_transcription_postprocess.py b/tests/test_rich_transcription_postprocess.py index 8e07de09eb..2cfc708ad8 100644 --- a/tests/test_rich_transcription_postprocess.py +++ b/tests/test_rich_transcription_postprocess.py @@ -32,3 +32,24 @@ def test_cough_and_sneeze_events_render_distinct_emojis(path, event, emoji): text = module.rich_transcription_postprocess(f"<|en|><|NEUTRAL|>{event}<|woitn|>hello") assert text == f"{emoji}hello" + + +@pytest.mark.parametrize("path", POSTPROCESS_UTILS, ids=lambda path: path.parts[-3]) +@pytest.mark.parametrize( + "first, second, expected", + [ + ("Hello world.", "How are you?", "Hello world. How are you?"), + ("so I went to the", "store", "so I went to the store"), + ("안녕하세요.", "감사합니다.", "안녕하세요. 감사합니다."), + ("你好。", "再见。", "你好。再见。"), + ("你好。", "Hello.", "你好。Hello."), + ], +) +def test_vad_segments_keep_a_space_between_spaced_scripts(path, first, second, expected): + module = _load(path) + tags = "<|NEUTRAL|><|Speech|><|withitn|>" + + # AutoModel joins the tagged text of each VAD segment with a single space. + text = module.rich_transcription_postprocess(f"<|en|>{tags}{first} <|en|>{tags}{second}") + + assert text == expected diff --git a/tests/test_runtime_eval_native.py b/tests/test_runtime_eval_native.py index e0e6154800..97133b3af8 100644 --- a/tests/test_runtime_eval_native.py +++ b/tests/test_runtime_eval_native.py @@ -8,6 +8,7 @@ import json import os from pathlib import Path +import shutil import subprocess import sys import wave @@ -275,6 +276,99 @@ def test_hotword_wrappers_decode_every_batch( assert model(paths, hotwords) == alone +@pytest.fixture(scope="module") +def timestamp_fixture(hotword_fixture, tmp_path_factory): + np = pytest.importorskip("numpy") + source, _ = hotword_fixture + directory = tmp_path_factory.mktemp("runtime-timestamp") + # The graphs only have to load: the timestamp test replaces the acoustic model's session. + for name, target in [ + ("tokens.json", "tokens.json"), + ("model_eb.onnx", "model_eb.onnx"), + ("model.onnx", "model.onnx"), + ("model_bb.torchscript", "model.torchscript"), + ]: + shutil.copy(source / name, directory / target) + (directory / "config.yaml").write_text( + json.dumps( + { + "frontend_conf": {"n_mels": 4, "lfr_m": 7, "lfr_n": 6, "dither": 0.0}, + "model_conf": {"predictor_bias": 1}, + } + ) + ) + (directory / "am.mvn").write_text( + " 28 28\n 1 [ " + "0 " * 28 + "]\n" + " 28 28\n 1 [ " + "1 " * 28 + "]\n" + ) + paths = [] + for seconds in (1, 3): + path = directory / f"tone {seconds}s.wav" + tone = np.sin(np.arange(16000 * seconds) * 2 * np.pi * 440 / 16000) * 3000 + with wave.open(str(path), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(16000) + output.writeframes(tone.astype(" + us_alphas = np.zeros((batch, 3 * feats.shape[1]), dtype=np.float32) + us_peaks = np.zeros_like(us_alphas) + for row, feat_len in enumerate(np.asarray(feats_len).tolist()): + frames = 3 * feat_len + us_alphas[row, :frames] = 3.0 / frames + us_peaks[row, np.linspace(5, frames - 6, 3).round().astype(int)] = 1.0 + return [am_scores, np.full(batch, 3, dtype=np.int32), us_alphas, us_peaks] + + +@pytest.mark.parametrize( + "backend, wrapper", + [ + ("onnx", "Paraformer"), + ("onnx", "ContextualParaformer"), + ("onnx", "SeacoParaformer"), + ("torch", "Paraformer"), + ], +) +def test_batched_timestamps_ignore_padding(timestamp_fixture, monkeypatch, backend, wrapper): + torch = pytest.importorskip("torch") + directory, paths = timestamp_fixture + location = "libtorch" if backend == "torch" else "onnxruntime" + monkeypatch.syspath_prepend(str(ROOT / "runtime/python" / location)) + module = importlib.import_module("funasr_" + backend + ".paraformer_bin") + model = getattr(module, wrapper)(str(directory), batch_size=2, intra_op_num_threads=1) + if backend == "torch": + model.ort_infer = lambda feats, feats_len: tuple( + torch.from_numpy(output) for output in timestamp_outputs(feats, feats_len) + ) + elif wrapper == "Paraformer": + model.ort_infer = lambda inputs: timestamp_outputs(*inputs) + else: + model.ort_infer_bb = lambda inputs: timestamp_outputs(*inputs[:2]) + hotwords = () if wrapper == "Paraformer" else ("魔搭",) + alone = [model(path, *hotwords)[0] for path in paths] + # The 1 s tone ends at 1000 ms, the 3 s tone at 3000 ms. + assert [row["timestamp"] for row in alone] == [ + [[70, 470], [470, 945]], + [[70, 670], [1450, 2925]], + ] + # Batched with the 3 s tone, the 1 s tone must not take on the padded frames. + assert model(paths, *hotwords) == alone + + @pytest.mark.parametrize("extension", ["flac", "ogg", "mp3"]) def test_onnx_wrappers_read_non_wav_paths(native_fixture, tmp_path, monkeypatch, extension): np = pytest.importorskip("numpy") @@ -310,3 +404,32 @@ def test_onnx_wrappers_read_non_wav_paths(native_fixture, tmp_path, monkeypatch, model = funasr_onnx.Paraformer(str(directory), device_id=DEVICE, intra_op_num_threads=1) assert model(str(audio)) == model(expected) assert {path.name: path.read_bytes() for path in tmp_path.iterdir()} == files + + +@pytest.mark.parametrize("backend", ["torch", "onnx"]) +@pytest.mark.parametrize("wrapper", ["ContextualParaformer", "SeacoParaformer"]) +def test_hotword_wrappers_skip_empty_and_cut_long_hotwords( + hotword_fixture, monkeypatch, backend, wrapper +): + directory, paths = hotword_fixture + location = "libtorch" if backend == "torch" else "onnxruntime" + monkeypatch.syspath_prepend(str(ROOT / "runtime/python" / location)) + module = importlib.import_module("funasr_" + backend + ".paraformer_bin") + model = getattr(module, wrapper)(str(directory), intra_op_num_threads=1) + # Like the C++ runtime: skip empty hotwords, and keep the first 10 characters of a longer + # one (the exported model_eb takes 10 per hotword). Ids: 1, 你 3, 好 4, 魔 5, 搭 6. + long_hotwords = " 魔搭 你好魔搭你好魔搭你好魔搭 " + matrix, positions = model.proc_hotword(long_hotwords) + assert matrix.tolist() == [ + [5, 6, 0, 0, 0, 0, 0, 0, 0, 0], + [3, 4, 5, 6, 3, 4, 5, 6, 3, 4], + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ] + assert positions.tolist() == [1, 9, 0] + matrix, positions = model.proc_hotword("") + assert matrix.tolist() == [[1, 0, 0, 0, 0, 0, 0, 0, 0, 0]] + assert positions.tolist() == [0] + # Both decode every file; the 12-character hotword used to stop the call with a ValueError. + for hotwords in (long_hotwords, ""): + result = model(paths, hotwords) + assert [row["preds"][0] for row in result] == ["你" * 4, "你" * 11, "你" * 17] diff --git a/tests/test_websocket_server_asr_model.py b/tests/test_websocket_server_asr_model.py new file mode 100644 index 0000000000..b1bab606fb --- /dev/null +++ b/tests/test_websocket_server_asr_model.py @@ -0,0 +1,64 @@ +"""Offline model options of runtime/python/websocket/funasr_wss_server.py. + +The README documents ``--asr_model`` for this server. The server parsed ``--asr_model`` and +``--asr_model_revision`` but built its offline model from a fixed "paraformer-zh" v2.0.4, +so both options had no effect. +""" + +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SERVER_PATH = ROOT / "runtime/python/websocket/funasr_wss_server.py" + + +def load_server(monkeypatch, *options): + """Run the server module with these command-line options; it builds its models on import.""" + pytest.importorskip("numpy") + # Each AutoModel call returns its keyword arguments. + funasr = types.ModuleType("funasr") + funasr.AutoModel = types.SimpleNamespace + monkeypatch.setitem(sys.modules, "funasr", funasr) + # Stand-ins for the modules the server uses to serve requests, not to build its models. + stand_ins = { + name: types.ModuleType(name) + for name in ("websockets", "torch", "scipy", "scipy.spatial", "scipy.spatial.distance") + } + stand_ins["scipy.spatial.distance"].cosine = None + for name, module in stand_ins.items(): + monkeypatch.setitem(sys.modules, name, module) + monkeypatch.setattr(sys, "argv", [str(SERVER_PATH), *options]) + spec = importlib.util.spec_from_file_location("funasr_wss_server_asr_model", SERVER_PATH) + server = importlib.util.module_from_spec(spec) + spec.loader.exec_module(server) + server.EXECUTOR.shutdown(wait=True) + return server + + +def test_default_offline_model_is_unchanged(monkeypatch): + server = load_server(monkeypatch) + assert server.model_asr.model == "paraformer-zh" + assert server.model_asr.model_revision == "v2.0.4" + + +@pytest.mark.parametrize( + "options, model, revision", + [ + (["--asr_model", "paraformer-en"], "paraformer-en", "v2.0.4"), + ( + ["--asr_model", "/models/my-paraformer", "--asr_model_revision", "master"], + "/models/my-paraformer", + "master", + ), + ], +) +def test_asr_model_options_select_the_offline_model(monkeypatch, options, model, revision): + server = load_server(monkeypatch, *options) + assert server.model_asr.model == model + assert server.model_asr.model_revision == revision + # The streaming model keeps its own options. + assert server.model_asr_streaming.model == server.args.asr_model_online diff --git a/web-pages/product-site/content/legacy-manifest.json b/web-pages/product-site/content/legacy-manifest.json index ddfe0410d0..20e7c98acf 100644 --- a/web-pages/product-site/content/legacy-manifest.json +++ b/web-pages/product-site/content/legacy-manifest.json @@ -37,9 +37,9 @@ "blog/lightweight-speech-recognition-cpu.html": "d6270066222ed5baef0df108e6a4155f4a87bb5482169f23f218794d371f3281", "blog/meeting-transcript-acceptance.html": "8173b3049ca707bdac46013748bc0ca7d9576ebf6f7ff6db6c940a833b12e4a9", "blog/punctuation-restoration-python.html": "063c5fe584bff780edc50f95214decbb3432326d4983334fcd8b38aabeb16128", - "blog/self-hosted-deepgram-assemblyai-alternative.html": "feb17aae8e0f3fe46914c8725eacc8fcdf7fe78bf22a291347d6107c24e3992e", + "blog/self-hosted-deepgram-assemblyai-alternative.html": "e7021f5a7f9bc0c40d88ff46ee53cd0d08cb5cbf17ae7aa8f9e8d85a15600c5f", "blog/self-hosted-google-aws-azure-speech-to-text.html": "8e4000fb9e2b33a48a3907d4bd0f55db57ba74121a89b5fa12b412c3c963f49a", - "blog/self-hosted-openai-whisper-api-alternative.html": "254184cd753f30e308329c301ae47e8f5f5dfb57c01d0abf3ce341034f2990f9", + "blog/self-hosted-openai-whisper-api-alternative.html": "f75f4b2fad03f6d1215347fbc98c71de1a59ff3c97cecc4aaf19bbb94e8eb777", "blog/sensevoice-deployment-guide.html": "61c34fb7626444a0f1b5c93013dd95c10c576333791eaca1ab819baf0a8cdad6", "blog/sensevoice-emotion-language-detection.html": "2451769a28af67788456aa02dad5201d218ac53fb84764cd7fbc32f991c953f8", "blog/speech-to-text-python-transcribe-audio.html": "87335710235dfdec9e01288f03a9a294c7bc3f730110e8d06f2b4c9d4906e284", @@ -80,9 +80,9 @@ "en/blog/lightweight-speech-recognition-cpu.html": "0e02c0e0853b12613d32c250f593c05c2052f18231207b63ee01d6e6600a86f7", "en/blog/meeting-transcript-acceptance.html": "ffc6eb2329267515bb367f21bb2d1c0523ff90d3f02b2e734ae41f78b02d0515", "en/blog/punctuation-restoration-python.html": "5853658aa8e936eda988fa087cba4a297de8044f979f6a9deef49206b8967a79", - "en/blog/self-hosted-deepgram-assemblyai-alternative.html": "4239a93246726c6f010a2fb20208211e050792a372e3ea356d1a93ac028a8a96", + "en/blog/self-hosted-deepgram-assemblyai-alternative.html": "49a5e6e8134c33f9951e3cc052571e8d8441acaa8522be233552f30b5f5410fa", "en/blog/self-hosted-google-aws-azure-speech-to-text.html": "4a8453451722b3ec8735224a586744ddfbfec9458cedd433f9fc908351027e9a", - "en/blog/self-hosted-openai-whisper-api-alternative.html": "14bdb877adfd113eeb04b215ffa015c29d8323859eeddd00d9e03d166dfa0473", + "en/blog/self-hosted-openai-whisper-api-alternative.html": "a95bde0e83ebea941144e5a4bd339ea866feee099f047d4b4fb01072c293cb70", "en/blog/sensevoice-deployment-guide.html": "8882912c6a84ec0a0e025d0eff8fcfa74bf76324bd4c9cae862c43c87c6e10b3", "en/blog/sensevoice-emotion-language-detection.html": "8f1432a4c07dab895dfda144d4fe1f64a487c72922febb6a379a15b7215e42a3", "en/blog/speech-to-text-python-transcribe-audio.html": "4fc57aef3c6177e28afa26fa6c41f67280b986e384b32a63a2f79ca67bcfe631", diff --git a/web-pages/product-site/legacy/blog/self-hosted-deepgram-assemblyai-alternative.html b/web-pages/product-site/legacy/blog/self-hosted-deepgram-assemblyai-alternative.html index 631379ae93..a92b9e038e 100644 --- a/web-pages/product-site/legacy/blog/self-hosted-deepgram-assemblyai-alternative.html +++ b/web-pages/product-site/legacy/blog/self-hosted-deepgram-assemblyai-alternative.html @@ -5,7 +5,7 @@