From c193425718d618e47b90c7642cad640e4caec15a Mon Sep 17 00:00:00 2001 From: spapi Date: Fri, 3 Jul 2026 13:01:31 +0200 Subject: [PATCH 1/4] Add support for FixedWords StreamAtt history selection for character-level languages --- config/seamless_streamatt_char.yaml | 13 ++++++++++++ .../speech_processors/base_streamatt.py | 21 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 config/seamless_streamatt_char.yaml diff --git a/config/seamless_streamatt_char.yaml b/config/seamless_streamatt_char.yaml new file mode 100644 index 0000000..bb93cfa --- /dev/null +++ b/config/seamless_streamatt_char.yaml @@ -0,0 +1,13 @@ +type: "simulstream.server.speech_processors.seamless_streamatt.SeamlessStreamAtt" +text_history: + type: "simulstream.server.speech_processors.base_streamatt.FixedCharsTextHistory" + history_chars: 20 +audio_history_max_duration: 360 # Maximum length for the audio buffer, in seconds +text_history_max_len: 128 +hf_model_name: "facebook/hf-seamless-m4t-medium" +seamless_version: 1 +speech_chunk_size: 0.5 # seconds +cross_attn_layer: 3 +cutoff_frame_num: 4 +detokenizer_type: "hf" +word_level_postprocess: False # Must be False for character-level languages diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py index fa7ccd4..3d31ed8 100644 --- a/simulstream/server/speech_processors/base_streamatt.py +++ b/simulstream/server/speech_processors/base_streamatt.py @@ -297,6 +297,27 @@ def select_text_history(self, text_history: List[str]): return new_history[::-1] +class FixedCharsTextHistory: + """ + Character-count-based textual history selection method that retains a pre-defined number of + tokens in the history (*history_chars*). + + Recommended for character-level languages (e.g., Chinese, Japanese) where word-boundary + markers (▁) are sparse, making :class:`FixedWordsTextHistory` ineffective: when few tokens + carry a BOW prefix, the word counter never reaches *history_words*, so the history is never + trimmed and the audio history grows without bound, causing AlignAtt to cut all new tokens. + + Args: + config (SimpleNamespace): Configuration object with an optional attribute: + - **history_chars (int)**: Number of tokens to retain. Defaults to 20. + """ + def __init__(self, config: SimpleNamespace): + self.history_chars = getattr(config, "history_chars", 20) + + def select_text_history(self, text_history: List[str]) -> List[str]: + return text_history[-self.history_chars:] + + class PunctuationTextHistory: """ Punctuation textual history selection method that retains the sentence From 30e441a361d19ff16aa07d1aad69bd542570dd84 Mon Sep 17 00:00:00 2001 From: spapi Date: Fri, 3 Jul 2026 13:56:00 +0200 Subject: [PATCH 2/4] Adding UTs for FixedWords and FixedChars --- uts/speech_processors/test_streamatt.py | 40 ++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/uts/speech_processors/test_streamatt.py b/uts/speech_processors/test_streamatt.py index 180c408..d943501 100644 --- a/uts/speech_processors/test_streamatt.py +++ b/uts/speech_processors/test_streamatt.py @@ -15,7 +15,45 @@ import unittest from types import SimpleNamespace -from simulstream.server.speech_processors.base_streamatt import PunctuationTextHistory +from simulstream.server.speech_processors.base_streamatt import ( + FixedCharsTextHistory, + FixedWordsTextHistory, + PunctuationTextHistory, +) + + +class TestFixedWordsTextHistory(unittest.TestCase): + def setUp(self): + self.history = FixedWordsTextHistory(SimpleNamespace(history_words=3)) + + def test_word_level(self): + """ Trims history to the last 3 BOW-started words in space-separated languages. """ + en_history = ["▁I", "▁am", "▁going", "▁to", "▁New", "▁York"] + self.assertEqual( + self.history.select_text_history(en_history), + ["▁going", "▁to", "▁New", "▁York"]) + + def test_word_level_with_subwords(self): + """ Subword continuations (no ▁) are included in the retained word. """ + en_history = ["▁inter", "nation", "al", "▁meet", "ing", "▁today"] + self.assertEqual( + self.history.select_text_history(en_history), + ["▁inter", "nation", "al", "▁meet", "ing", "▁today"]) + + +class TestFixedCharsTextHistory(unittest.TestCase): + def setUp(self): + self.history = FixedCharsTextHistory(SimpleNamespace(history_chars=3)) + + def test_char_level(self): + """ Keeps the last 3 tokens for character-level languages. """ + zh_history = ['大', '家', '好', '我', '在', '谈', '论'] + self.assertEqual(self.history.select_text_history(zh_history), ['在', '谈', '论']) + + def test_shorter_than_limit(self): + """ Returns the full history when it is shorter than history_chars. """ + zh_history = ['大', '家'] + self.assertEqual(self.history.select_text_history(zh_history), ['大', '家']) class TestPunctuationTextHistory(unittest.TestCase): From 39bd3b4a0976b5fb54b2301543c19375d0acf102 Mon Sep 17 00:00:00 2001 From: spapi Date: Fri, 3 Jul 2026 14:51:08 +0200 Subject: [PATCH 3/4] Fix UT --- uts/speech_processors/test_streamatt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uts/speech_processors/test_streamatt.py b/uts/speech_processors/test_streamatt.py index d943501..a20878f 100644 --- a/uts/speech_processors/test_streamatt.py +++ b/uts/speech_processors/test_streamatt.py @@ -31,7 +31,7 @@ def test_word_level(self): en_history = ["▁I", "▁am", "▁going", "▁to", "▁New", "▁York"] self.assertEqual( self.history.select_text_history(en_history), - ["▁going", "▁to", "▁New", "▁York"]) + ["▁to", "▁New", "▁York"]) def test_word_level_with_subwords(self): """ Subword continuations (no ▁) are included in the retained word. """ From d2a69d9927230a43f753fe2f30a69c4320732244 Mon Sep 17 00:00:00 2001 From: spapi Date: Fri, 3 Jul 2026 18:57:46 +0200 Subject: [PATCH 4/4] Address comment --- config/seamless_streamatt_char.yaml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 config/seamless_streamatt_char.yaml diff --git a/config/seamless_streamatt_char.yaml b/config/seamless_streamatt_char.yaml deleted file mode 100644 index bb93cfa..0000000 --- a/config/seamless_streamatt_char.yaml +++ /dev/null @@ -1,13 +0,0 @@ -type: "simulstream.server.speech_processors.seamless_streamatt.SeamlessStreamAtt" -text_history: - type: "simulstream.server.speech_processors.base_streamatt.FixedCharsTextHistory" - history_chars: 20 -audio_history_max_duration: 360 # Maximum length for the audio buffer, in seconds -text_history_max_len: 128 -hf_model_name: "facebook/hf-seamless-m4t-medium" -seamless_version: 1 -speech_chunk_size: 0.5 # seconds -cross_attn_layer: 3 -cutoff_frame_num: 4 -detokenizer_type: "hf" -word_level_postprocess: False # Must be False for character-level languages