From cafb60913fc27fce0b8ce7615486d6f127d41b92 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 13:05:23 +0100
Subject: [PATCH 001/157] DOA Policy first implementation

---
 config/phi4multimodal_doa.yaml                |  11 ++
 .../server/speech_processors/base_doa.py      | 142 ++++++++++++++++
 .../speech_processors/base_streamatt.py       |  23 +--
 .../speech_processors/phi4multimodal_doa.py   | 156 ++++++++++++++++++
 4 files changed, 321 insertions(+), 11 deletions(-)
 create mode 100644 config/phi4multimodal_doa.yaml
 create mode 100644 simulstream/server/speech_processors/base_doa.py
 create mode 100644 simulstream/server/speech_processors/phi4multimodal_doa.py

diff --git a/config/phi4multimodal_doa.yaml b/config/phi4multimodal_doa.yaml
new file mode 100644
index 0000000..7057093
--- /dev/null
+++ b/config/phi4multimodal_doa.yaml
@@ -0,0 +1,11 @@
+type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
+  history_words: 10
+audio_history_max_duration: 360  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 0.5  # seconds
+cross_attn_layer: 3
+cutoff_frame_num: 4
+detokenizer_type: "hf"
+word_level_postprocess: True  # Disable if character-level language
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
new file mode 100644
index 0000000..cc54ec2
--- /dev/null
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -0,0 +1,142 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import logging
+from abc import abstractmethod
+from types import SimpleNamespace
+from typing import List, Tuple
+
+import numpy as np
+import torch
+
+from simulstream.server.speech_processors import SAMPLE_RATE
+from simulstream.server.speech_processors.base_streamatt import BaseStreamAtt
+
+logger = logging.getLogger(__name__)
+
+
+TEMPLATED_SPEECH_PROMPT = \
+    ("You are a professional {src_lang}-to-{tgt_lang} translator. Your goal is to accurately "
+     "convey the meaning and nuances of the original {src_lang} speech while adhering to "
+     "{tgt_lang} grammar, vocabulary, and cultural sensitivities. Use precise terminology and a "
+     "tone appropriate for academic or instructional materials. Produce only the {tgt_lang} "
+     "translation, without any additional explanations or commentary. Please translate the "
+     "provided {src_lang} speech into {tgt_lang}:")
+
+LANG_MAPPER = {"en": "English", "it": "Italian"}
+
+
+class DecoderOnlyAttention(BaseStreamAtt):
+    """
+    Generic Decoder-only Attention-based policy for SpeechLLMs.
+
+    The class handles:
+    - Rolling raw-waveform history accumulation.
+    - Greedy generation with ``output_attentions=True``.
+    - Building the proxy cross-attention matrix from self-attention weights.
+    - Token decoding.
+
+    Subclasses must implement the five abstract methods listed below.
+
+    Parameters
+    ----------
+    config : SimpleNamespace
+        All fields from :class:`BaseStreamAtt`, plus:
+
+        device : str
+            Torch device string.  Default: ``"cuda"``.
+        audio_max_frames : int
+            Maximum raw waveform samples to keep in the rolling history
+            (at 16 kHz).  Default: ``480_000`` (30 s).
+        max_new_tokens : int
+            Maximum tokens to generate per chunk.  Default: ``200``.
+    """
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.cross_attn_layer = getattr(self.config, "attention_layer", 3)
+        self.max_new_tokens = getattr(self.config, "max_new_tokens", 4096)
+        self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 360)
+        self.src_lang_tag = getattr(self.config, "src_lang_tag", "en")
+        self.tgt_lang_tag = getattr(self.config, "tgt_lang_tag", "en")
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    @property
+    def audio_max_len(self) -> int:
+        """Maximum raw-waveform samples to keep in the rolling audio history."""
+        return self.audio_history_max_duration * SAMPLE_RATE
+
+    @abstractmethod
+    def load_model(self, config: SimpleNamespace) -> None:
+        """
+        Load the model and processor from *config* and assign them to
+        ``self.model`` and ``self.processor``.
+
+        The model **must** be loaded with ``output_attentions=True`` (or the
+        equivalent flag for the architecture) and
+        ``_attn_implementation="eager"``.
+        """
+        ...
+
+    @abstractmethod
+    def build_prompt(self) -> str:
+        """
+        Return the prompt string to be used with audio tokens.
+        """
+        ...
+
+    @abstractmethod
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        """
+        Given the *entire* rolling waveform history (float32, 16 kHz), return
+        a ``dict`` of ``torch.Tensor`` inputs ready to be passed to
+        ``self.model.generate(**inputs, …)``.
+
+        The tensors must already be on ``self.device``.
+        """
+        ...
+
+    @abstractmethod
+    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+        """
+        Generate tokens from the given inputs together with the self-attention scores.
+
+        Returns:
+            Tuple[List[str], torch.Tensor]:
+                List[str]: A list of generated tokens.
+                torch.Tensor: Self-attention scores between speech and text with dimension
+                (token_len, audio_len).
+        """
+        ...
+
+    @abstractmethod
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert a list of decoded tokens to a plain output string."""
+        ...
+
+    def _preprocess(self, waveform: np.float32) -> dict:
+        """
+        Append *waveform* to ``self.audio_history``, enforce the maximum length,
+        and delegate to :meth:`build_processor_inputs`.
+        """
+        if self.audio_history is None:
+            self.audio_history = waveform
+        else:
+            self.audio_history = np.concatenate([self.audio_history, waveform])
+
+        if len(self.audio_history) > self.audio_max_len:
+            logger.warning("Audio history exceeded %d samples; trimming.", self.audio_max_len)
+            self.audio_history = self.audio_history[-self.audio_max_len:]
+
+        return self.build_processor_inputs(self.audio_history)
diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index fa7ccd4..249c76a 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -73,9 +73,10 @@ class BaseStreamAtt(BaseSpeechProcessor):
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
         self.config = config
-        text_history_config = self.config.text_history
-        text_history_cls = class_load(text_history_config.type)
-        self.text_history_method = text_history_cls(text_history_config)
+        self.text_history_config = self.config.text_history
+        text_history_cls = class_load(self.text_history_config.type)
+        self.bow_prefix = getattr(self.config, "bow_prefix", BOW_PREFIX)
+        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = getattr(self.config, "audio_subsampling_factor", 1)
         self.text_history_max_len = getattr(self.config, "text_history_max_len", 128)
         self.cross_attn_layer = getattr(self.config, "cross_attention_layer", 3)
@@ -182,8 +183,7 @@ def _update_speech_history(self, discarded_text: int, cross_attn: torch.Tensor)
         # Check audio history not exceeding maximum allowed length
         self._cut_audio_exceeding_maxlen()
 
-    @staticmethod
-    def _strip_incomplete_words(tokens: List[str]) -> List[str]:
+    def _strip_incomplete_words(self, tokens: List[str]) -> List[str]:
         """
         Remove last incomplete word(s) from the new hypothesis.
 
@@ -198,7 +198,7 @@ def _strip_incomplete_words(tokens: List[str]) -> List[str]:
         num_tokens_incomplete = 0
         for tok in reversed(tokens):
             num_tokens_incomplete += 1
-            if tok.startswith(BOW_PREFIX):
+            if tok.startswith(self.bow_prefix):
                 # slice off the trailing incomplete tokens
                 tokens_to_write = tokens[:-num_tokens_incomplete]
                 break
@@ -276,8 +276,9 @@ class FixedWordsTextHistory:
 
     The current implementation supports only SentencePiece.
     """
-    def __init__(self, config: SimpleNamespace):
+    def __init__(self, config: SimpleNamespace, bow_prefix: str):
         self.history_words = getattr(config, "history_words", 20)
+        self.bow_prefix = bow_prefix
         self.config = config
 
     def select_text_history(self, text_history: List[str]):
@@ -285,9 +286,9 @@ def select_text_history(self, text_history: List[str]):
         new_history = []
         for token in reversed(text_history):
             new_history.append(token)
-            # Check if 'BOW_PREFIX' (space in SentencePiece) is contained in the token,
-            # meaning that we reached the beginning of the word that should be counted
-            if BOW_PREFIX in token:
+            # Check if bow_prefix is contained in the token, meaning that we reached
+            # the beginning of the word that should be counted
+            if self.bow_prefix in token:
                 words_to_keep -= 1
                 # When all the words to keep are consumed, the accumulation is stopped
                 # and the prefix is returned
@@ -307,7 +308,7 @@ class PunctuationTextHistory:
 
     STRONG_PUNCTUATION = [".", "!", "?", ":", ";", "。"]
 
-    def __init__(self, config: SimpleNamespace):
+    def __init__(self, config: SimpleNamespace, bow_prefix: str):
         self.config = config
 
     def select_text_history(self, text_history):
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
new file mode 100644
index 0000000..d234069
--- /dev/null
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -0,0 +1,156 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import torch
+import numpy as np
+
+from types import SimpleNamespace
+from typing import List, Tuple
+
+from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
+
+from simulstream.server.speech_processors import SAMPLE_RATE, class_load
+from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, TEMPLATED_SPEECH_PROMPT, LANG_MAPPER
+
+
+class Phi4MultimodalDOA(DecoderOnlyAttention):
+    """
+    Decoder-Only Attention agent for ``microsoft/Phi-4-multimodal-instruct``.
+
+    Extra config fields
+    -------------------
+    model_path : str
+        Default: ``"microsoft/Phi-4-multimodal-instruct"``
+    task : str
+        ``"transcribe"`` or ``"translate"``.  Default: ``"transcribe"``
+    target_lang : str
+        Target language when ``task="translate"``.  Default: ``"English"``
+    """
+
+    # Phi-4 special tokens
+    _USER_START  = "<|user|>"
+    _AUDIO_TOKEN = "<|audio_1|>"
+    _END_TOKEN   = "<|end|>"
+    _ASST_START  = "<|assistant|>"
+
+    BOW_PREFIX = " "
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.bow_prefix = self.BOW_PREFIX
+        text_history_cls = class_load(self.text_history_config.type)
+        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+
+    def load_model(self, config: SimpleNamespace) -> None:
+        model_path = "microsoft/Phi-4-multimodal-instruct"
+
+        self.processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_path,
+            device_map="cuda",
+            torch_dtype="auto",
+            trust_remote_code=True,
+            _attn_implementation="eager",
+        )
+        self.model.eval()
+        self.generation_config = GenerationConfig.from_pretrained(model_path)
+
+    @property
+    def audio_max_len(self) -> int:
+        return getattr(self.config, "audio_max_frames", 480_000)
+
+    def build_prompt(self) -> str:
+        filled_prompt = (
+            TEMPLATED_SPEECH_PROMPT
+            .replace("{src_lang}", LANG_MAPPER[self.src_lang_tag])
+            .replace("{tgt_lang}", LANG_MAPPER[self.tgt_lang_tag]))
+        prefix = self.text_history if self.text_history else ""
+        return (
+            f"{self._USER_START}{self._AUDIO_TOKEN}"
+            f"{filled_prompt}{self._END_TOKEN}"
+            f"{self._ASST_START}{prefix}"
+        )
+
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        return self.processor(
+            text=self.build_prompt(),
+            audios=[(waveform, SAMPLE_RATE)],
+            return_tensors="pt",
+        ).to(self.device)
+
+    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+        """
+        Run greedy generation and build the proxy cross-attention matrix.
+
+        ``output.attentions`` layout (decoder-only, use_cache=True)
+        ────────────────────────────────────────────────────────────
+        output.attentions[0][layer]  → (1, H, input_len, input_len)  ← prefill
+        output.attentions[i][layer]  → (1, H, 1, input_len+i)        ← new token i
+
+        Returns
+        -------
+        List[str]
+            A list of generated tokens (n_new).
+        torch.Tensor
+            Proxy cross-attention scores extracted from the self-attention scores,
+            averaged over heads at ``self.cross_attn_layer`` (prefix + n_new, audio_len).
+        """
+        input_ids = inputs["input_ids"]  # (1, input_len)
+        input_len = input_ids.shape[1]
+
+        # Locate audio positions ──────────────────────────────────────────────────────────────────
+        AUDIO_SPECIAL_TOKEN_ID = 200011  # _AUDIO_SPECIAL_TOKEN_ID in modeling_phi4mm.py
+        audio_positions = (input_ids[0] == AUDIO_SPECIAL_TOKEN_ID).nonzero(as_tuple=True)[0]
+        audio_len = audio_positions.shape[0]
+
+        # Generate ────────────────────────────────────────────────────────────────────────────────
+        output = self.model.generate(
+            **inputs,
+            max_new_tokens=self.max_new_tokens,
+            generation_config=self.generation_config,
+            output_attentions=True,
+            return_dict_in_generate=True,
+            do_sample=False,
+        )
+
+        # Decode newly generated tokens only ──────────────────────────────────────────────────────
+        new_ids = output.sequences[:, input_len:]  # (1, n_new)
+        new_tokens = [
+            self.processor.tokenizer.decode([t], skip_special_tokens=False)
+            for t in new_ids[0]
+        ]
+
+        # Build proxy cross-attention for the hypothesis (prefix + new_tokens) ────────────────────
+        # Prefix rows from the prefill pass
+        # output.attentions[0][layer]: (1, H, input_len, input_len)
+        prefill_attn = (output.attentions[0][self.cross_attn_layer][0]
+                        .mean(dim=0))  # (input_len, input_len)
+        prefix_rows = prefill_attn[
+            self.cross_attn_layer:, :][:, audio_positions]  # (n_prefix, audio_len)
+        # New-token rows: one per step, each (1, H, 1, input_len+i)
+        new_rows = [
+            step_attn[self.cross_attn_layer][0]
+            .mean(dim=0).squeeze(0)[audio_positions]  # (audio_len,)
+            for step_attn in output.attentions[1:]
+        ]
+        new_attn = torch.stack(new_rows, dim=0) if new_rows else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
+
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0) # (n_prefix + n_new, audio_len)
+
+        return new_tokens, cross_attn
+
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens).strip()

From 4f9dbd165ae6624e92532f11af194dfbc6e8ee98 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 13:09:42 +0100
Subject: [PATCH 002/157] Improve Phi4-Multimodal DOA description

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index d234069..f20a6a2 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -93,15 +93,16 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         """
         Run greedy generation and build the proxy cross-attention matrix.
 
-        ``output.attentions`` layout (decoder-only, use_cache=True)
-        ────────────────────────────────────────────────────────────
+        ``output.attentions`` (use_cache=True) contains the self-attention scores,
+        for each step and layer. H is the dimension of the attention heads.
+        ───────────────────────────────────────────────────────────────────────────────────────────
         output.attentions[0][layer]  → (1, H, input_len, input_len)  ← prefill
         output.attentions[i][layer]  → (1, H, 1, input_len+i)        ← new token i
 
         Returns
         -------
         List[str]
-            A list of generated tokens (n_new).
+            A list of the newly generated tokens (n_new).
         torch.Tensor
             Proxy cross-attention scores extracted from the self-attention scores,
             averaged over heads at ``self.cross_attn_layer`` (prefix + n_new, audio_len).

From 99282e25d30604275502eee46ae78ad4c9f14b5f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 16:11:37 +0100
Subject: [PATCH 003/157] Rename DOA Phi config

---
 .../{phi4multimodal_doa.py => phi4multimodal_doa_frame4.py}     | 2 --
 1 file changed, 2 deletions(-)
 rename simulstream/server/speech_processors/{phi4multimodal_doa.py => phi4multimodal_doa_frame4.py} (98%)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa_frame4.py
similarity index 98%
rename from simulstream/server/speech_processors/phi4multimodal_doa.py
rename to simulstream/server/speech_processors/phi4multimodal_doa_frame4.py
index f20a6a2..76f1df9 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa_frame4.py
@@ -32,8 +32,6 @@ class Phi4MultimodalDOA(DecoderOnlyAttention):
     -------------------
     model_path : str
         Default: ``"microsoft/Phi-4-multimodal-instruct"``
-    task : str
-        ``"transcribe"`` or ``"translate"``.  Default: ``"transcribe"``
     target_lang : str
         Target language when ``task="translate"``.  Default: ``"English"``
     """

From 9dd766893d51b73950a2c49e3ff305a743006002 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 16:13:04 +0100
Subject: [PATCH 004/157] Rename DOA Phi config

---
 .../{phi4multimodal_doa.yaml => phi4multimodal_doa_frame4.yaml}   | 0
 .../{phi4multimodal_doa_frame4.py => phi4multimodal_doa.py}       | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename config/{phi4multimodal_doa.yaml => phi4multimodal_doa_frame4.yaml} (100%)
 rename simulstream/server/speech_processors/{phi4multimodal_doa_frame4.py => phi4multimodal_doa.py} (100%)

diff --git a/config/phi4multimodal_doa.yaml b/config/phi4multimodal_doa_frame4.yaml
similarity index 100%
rename from config/phi4multimodal_doa.yaml
rename to config/phi4multimodal_doa_frame4.yaml
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa_frame4.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
similarity index 100%
rename from simulstream/server/speech_processors/phi4multimodal_doa_frame4.py
rename to simulstream/server/speech_processors/phi4multimodal_doa.py

From f841ce2f65bab4f5fd02c0f77aa0761dfdf66e6e Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 16:19:47 +0100
Subject: [PATCH 005/157] Correct audio subsampling factor

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 76f1df9..c86e68e 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -43,12 +43,15 @@ class Phi4MultimodalDOA(DecoderOnlyAttention):
     _ASST_START  = "<|assistant|>"
 
     BOW_PREFIX = " "
+    ENCODER_SUBSAMPLING_FACTOR = 8
+    HOP_LENGTH = 160    # 10ms at 16kHz
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
         self.bow_prefix = self.BOW_PREFIX
         text_history_cls = class_load(self.text_history_config.type)
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.audio_subsampling_factor = self.ENCODER_SUBSAMPLING_FACTOR * self.HOP_LENGTH
 
     def load_model(self, config: SimpleNamespace) -> None:
         model_path = "microsoft/Phi-4-multimodal-instruct"

From 73fed6b9a10aaa357a18d706e9427bb89e9d86b7 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 16:21:33 +0100
Subject: [PATCH 006/157] Fix load_model to class method

---
 .../server/speech_processors/phi4multimodal_doa.py    | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index c86e68e..c4cc679 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -53,19 +53,20 @@ def __init__(self, config: SimpleNamespace):
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.ENCODER_SUBSAMPLING_FACTOR * self.HOP_LENGTH
 
-    def load_model(self, config: SimpleNamespace) -> None:
+    @classmethod
+    def load_model(cls, config: SimpleNamespace) -> None:
         model_path = "microsoft/Phi-4-multimodal-instruct"
 
-        self.processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
-        self.model = AutoModelForCausalLM.from_pretrained(
+        cls.processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+        cls.model = AutoModelForCausalLM.from_pretrained(
             model_path,
             device_map="cuda",
             torch_dtype="auto",
             trust_remote_code=True,
             _attn_implementation="eager",
         )
-        self.model.eval()
-        self.generation_config = GenerationConfig.from_pretrained(model_path)
+        cls.model.eval()
+        cls.generation_config = GenerationConfig.from_pretrained(model_path)
 
     @property
     def audio_max_len(self) -> int:

From c456013df13b6eee95eb0026e1087dc6854ba211 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 16:26:19 +0100
Subject: [PATCH 007/157] Fix lang init

---
 simulstream/server/speech_processors/base_doa.py           | 6 ++++++
 simulstream/server/speech_processors/phi4multimodal_doa.py | 4 ++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index cc54ec2..8f17b30 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -125,6 +125,12 @@ def tokens_to_string(self, tokens: List[str]) -> str:
         """Convert a list of decoded tokens to a plain output string."""
         ...
 
+    def set_target_language(self, language: str) -> None:
+        self.tgt_lang = language
+
+    def set_source_language(self, language: str) -> None:
+        self.src_lang = language
+
     def _preprocess(self, waveform: np.float32) -> dict:
         """
         Append *waveform* to ``self.audio_history``, enforce the maximum length,
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index c4cc679..bf75dde 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -75,8 +75,8 @@ def audio_max_len(self) -> int:
     def build_prompt(self) -> str:
         filled_prompt = (
             TEMPLATED_SPEECH_PROMPT
-            .replace("{src_lang}", LANG_MAPPER[self.src_lang_tag])
-            .replace("{tgt_lang}", LANG_MAPPER[self.tgt_lang_tag]))
+            .replace("{src_lang}", LANG_MAPPER[self.src_lang])
+            .replace("{tgt_lang}", LANG_MAPPER[self.tgt_lang]))
         prefix = self.text_history if self.text_history else ""
         return (
             f"{self._USER_START}{self._AUDIO_TOKEN}"

From 54752a962428750c6fa9c9991e8d6930d84aec71 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:03:51 +0100
Subject: [PATCH 008/157] Fix newer transformers compatibility

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index bf75dde..91b029b 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -122,6 +122,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             **inputs,
             max_new_tokens=self.max_new_tokens,
             generation_config=self.generation_config,
+            num_logits_to_keep=1,
             output_attentions=True,
             return_dict_in_generate=True,
             do_sample=False,

From 48ab026cb73ce56abce3aedc4b9fe9426560f757 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:10:23 +0100
Subject: [PATCH 009/157] Debug

---
 config/phi4multimodal_doa_frame4.yaml                  | 2 +-
 simulstream/server/speech_processors/base_streamatt.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_frame4.yaml
index 7057093..68a73fe 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_frame4.yaml
@@ -2,7 +2,7 @@ type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
   history_words: 10
-audio_history_max_duration: 360  # Maximum length for the audio buffer, in seconds
+audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 0.5  # seconds
 cross_attn_layer: 3
diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 249c76a..055f3de 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -248,8 +248,10 @@ def process_chunk(self, waveform: np.float32) -> IncrementalOutput:
         speech = self._preprocess(waveform)
         # Generate new hypothesis with its corresponding cross-attention scores (no prefix)
         generated_tokens, cross_attn = self._generate(speech)
+        print(generated_tokens, cross_attn.shape)
         # Select the part of the new hypothesis to be emitted, and trim cross-attention accordingly
         selected_output = self.alignatt_policy(generated_tokens, cross_attn)
+        print(selected_output)
         incremental_output = self._build_incremental_outputs(selected_output)
         # Discard textual history, if needed
         discarded_text = self._update_text_history(selected_output)

From 03091ac6742b0a00280c7340333b4c945111e5c0 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:20:00 +0100
Subject: [PATCH 010/157] Remove end of sentence

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 91b029b..e771734 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -131,7 +131,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         # Decode newly generated tokens only ──────────────────────────────────────────────────────
         new_ids = output.sequences[:, input_len:]  # (1, n_new)
         new_tokens = [
-            self.processor.tokenizer.decode([t], skip_special_tokens=False)
+            self.processor.tokenizer.decode([t], skip_special_tokens=True)
             for t in new_ids[0]
         ]
 
@@ -146,7 +146,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         new_rows = [
             step_attn[self.cross_attn_layer][0]
             .mean(dim=0).squeeze(0)[audio_positions]  # (audio_len,)
-            for step_attn in output.attentions[1:]
+            for step_attn in output.attentions[1:-1]    # avoid attention of <|end|> token
         ]
         new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)

From aa990eefd0ec3ee8da86beb6310b63aadaef1551 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:25:03 +0100
Subject: [PATCH 011/157] Disable eos

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index e771734..1001770 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -123,6 +123,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             max_new_tokens=self.max_new_tokens,
             generation_config=self.generation_config,
             num_logits_to_keep=1,
+            eos_token_id=None,
             output_attentions=True,
             return_dict_in_generate=True,
             do_sample=False,

From 6dc2e4d0e0f88d6491fcb486b3bb71e0ac32b2c9 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:26:44 +0100
Subject: [PATCH 012/157] Revert

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 1001770..e771734 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -123,7 +123,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             max_new_tokens=self.max_new_tokens,
             generation_config=self.generation_config,
             num_logits_to_keep=1,
-            eos_token_id=None,
             output_attentions=True,
             return_dict_in_generate=True,
             do_sample=False,

From 042e682b289aeaabfaa6fad080ba5a9010d05861 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:29:49 +0100
Subject: [PATCH 013/157] Try different EOS

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index e771734..95e5f69 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -52,6 +52,7 @@ def __init__(self, config: SimpleNamespace):
         text_history_cls = class_load(self.text_history_config.type)
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.ENCODER_SUBSAMPLING_FACTOR * self.HOP_LENGTH
+        self.eos_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|end|>")
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
@@ -126,6 +127,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output_attentions=True,
             return_dict_in_generate=True,
             do_sample=False,
+            bad_words_ids=[[self.eos_token_id]]
         )
 
         # Decode newly generated tokens only ──────────────────────────────────────────────────────

From 2dcfdcdb5cf38ec9fd0b5e109c90db0af96f0dbb Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:35:58 +0100
Subject: [PATCH 014/157] Debug

---
 simulstream/server/speech_processors/base_streamatt.py     | 4 +++-
 simulstream/server/speech_processors/phi4multimodal_doa.py | 2 --
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 055f3de..0a26436 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -251,10 +251,12 @@ def process_chunk(self, waveform: np.float32) -> IncrementalOutput:
         print(generated_tokens, cross_attn.shape)
         # Select the part of the new hypothesis to be emitted, and trim cross-attention accordingly
         selected_output = self.alignatt_policy(generated_tokens, cross_attn)
-        print(selected_output)
+        print(f"selected {selected_output}")
         incremental_output = self._build_incremental_outputs(selected_output)
         # Discard textual history, if needed
         discarded_text = self._update_text_history(selected_output)
+        print(f"text history {self.text_history}")
+        print(f"discarded {discarded_text}")
         # Trim audio corresponding to the discarded textual history
         self._update_speech_history(discarded_text, cross_attn)
         return incremental_output
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 95e5f69..e771734 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -52,7 +52,6 @@ def __init__(self, config: SimpleNamespace):
         text_history_cls = class_load(self.text_history_config.type)
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.ENCODER_SUBSAMPLING_FACTOR * self.HOP_LENGTH
-        self.eos_token_id = self.processor.tokenizer.convert_tokens_to_ids("<|end|>")
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
@@ -127,7 +126,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output_attentions=True,
             return_dict_in_generate=True,
             do_sample=False,
-            bad_words_ids=[[self.eos_token_id]]
         )
 
         # Decode newly generated tokens only ──────────────────────────────────────────────────────

From 46ad7f879b3572c04cfabda4d3291b58b7ae99ce Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:40:34 +0100
Subject: [PATCH 015/157] Debug

---
 simulstream/server/speech_processors/base_streamatt.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 0a26436..2b261a0 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -254,11 +254,13 @@ def process_chunk(self, waveform: np.float32) -> IncrementalOutput:
         print(f"selected {selected_output}")
         incremental_output = self._build_incremental_outputs(selected_output)
         # Discard textual history, if needed
-        discarded_text = self._update_text_history(selected_output)
         print(f"text history {self.text_history}")
+        discarded_text = self._update_text_history(selected_output)
         print(f"discarded {discarded_text}")
         # Trim audio corresponding to the discarded textual history
+        print(f"previous speech history {self.audio_history}")
         self._update_speech_history(discarded_text, cross_attn)
+        print(f"trimmed speech history {self.audio_history}")
         return incremental_output
 
     def end_of_stream(self) -> IncrementalOutput:

From bc37b8d7c33edf57c4c2dafe0585b851e5187122 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:41:23 +0100
Subject: [PATCH 016/157] Debug

---
 simulstream/server/speech_processors/base_streamatt.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 2b261a0..654c190 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -258,9 +258,9 @@ def process_chunk(self, waveform: np.float32) -> IncrementalOutput:
         discarded_text = self._update_text_history(selected_output)
         print(f"discarded {discarded_text}")
         # Trim audio corresponding to the discarded textual history
-        print(f"previous speech history {self.audio_history}")
+        print(f"previous speech history {self.audio_history.shape}")
         self._update_speech_history(discarded_text, cross_attn)
-        print(f"trimmed speech history {self.audio_history}")
+        print(f"trimmed speech history {self.audio_history.shape}")
         return incremental_output
 
     def end_of_stream(self) -> IncrementalOutput:

From b8a5e3315b5469e8576654766dad66a23dc70b6b Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 17:55:51 +0100
Subject: [PATCH 017/157] Partial fix

---
 .../server/speech_processors/phi4multimodal_doa.py       | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index e771734..03a77dc 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -77,7 +77,7 @@ def build_prompt(self) -> str:
             TEMPLATED_SPEECH_PROMPT
             .replace("{src_lang}", LANG_MAPPER[self.src_lang])
             .replace("{tgt_lang}", LANG_MAPPER[self.tgt_lang]))
-        prefix = self.text_history if self.text_history else ""
+        prefix = "".join(self.text_history) if self.text_history else ""
         return (
             f"{self._USER_START}{self._AUDIO_TOKEN}"
             f"{filled_prompt}{self._END_TOKEN}"
@@ -140,8 +140,11 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         # output.attentions[0][layer]: (1, H, input_len, input_len)
         prefill_attn = (output.attentions[0][self.cross_attn_layer][0]
                         .mean(dim=0))  # (input_len, input_len)
-        prefix_rows = prefill_attn[
-            self.cross_attn_layer:, :][:, audio_positions]  # (n_prefix, audio_len)
+        prefix_len = len(self.text_history) if self.text_history else 0
+        if prefix_len > 0:
+            prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
+        else:
+            prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
         # New-token rows: one per step, each (1, H, 1, input_len+i)
         new_rows = [
             step_attn[self.cross_attn_layer][0]

From aa41e41abe24ad1bd90a00d96e88f3a051242dcd Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:06:06 +0100
Subject: [PATCH 018/157] Increase stability

---
 config/phi4multimodal_doa_frame4.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_frame4.yaml
index 68a73fe..d118a52 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_frame4.yaml
@@ -4,7 +4,7 @@ text_history:
   history_words: 10
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
-speech_chunk_size: 0.5  # seconds
+speech_chunk_size: 1  # seconds
 cross_attn_layer: 3
 cutoff_frame_num: 4
 detokenizer_type: "hf"

From 5d08e7dc26def1da7d19fdf9eefe1b6bacdda4ad Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:25:46 +0100
Subject: [PATCH 019/157] Add cross attention normalization

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 03a77dc..df966ba 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -155,6 +155,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             torch.zeros(0, max(audio_len, 1), device=self.device)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0) # (n_prefix + n_new, audio_len)
+        cross_attn = self.normalize_attn(cross_attn)
 
         return new_tokens, cross_attn
 

From d9def94e8c5f9fba95d7873a99e1c41a66969db8 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:28:30 +0100
Subject: [PATCH 020/157] revert speech chunk

---
 config/phi4multimodal_doa_frame4.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_frame4.yaml
index d118a52..68a73fe 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_frame4.yaml
@@ -4,7 +4,7 @@ text_history:
   history_words: 10
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
-speech_chunk_size: 1  # seconds
+speech_chunk_size: 0.5  # seconds
 cross_attn_layer: 3
 cutoff_frame_num: 4
 detokenizer_type: "hf"

From 87ee050cd586f28fe757b4cdde64b69babdf1095 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:30:59 +0100
Subject: [PATCH 021/157] debug

---
 .../server/speech_processors/base_streamatt.py       | 12 ++++++++++++
 .../server/speech_processors/phi4multimodal_doa.py   | 11 ++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 654c190..ab6b7ef 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -176,6 +176,18 @@ def _update_speech_history(self, discarded_text: int, cross_attn: torch.Tensor)
 
         # Multiply by the subsampling factor to recover the original number of frames
         frames_to_cut = earliest_attended_idx * self.audio_subsampling_factor
+        print(
+            "streamatt trim debug",
+            {
+                "discarded_text": int(discarded_text),
+                "history_len": int(len(self.text_history)),
+                "cross_attn_rows": int(cross_attn.shape[0]),
+                "cross_attn_cols": int(cross_attn.shape[1]),
+                "most_attended_head": most_attended_idxs[:5].tolist(),
+                "earliest_attended_idx": int(earliest_attended_idx),
+                "frames_to_cut": int(frames_to_cut),
+            },
+        )
 
         # Cut the unattended audio features
         self.audio_history = self.audio_history[frames_to_cut:]
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index df966ba..8357d42 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -145,6 +145,16 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
         else:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
+        print(
+            "phi4 prefix debug",
+            {
+                "history_len": prefix_len,
+                "input_len": int(input_len),
+                "audio_len": int(audio_len),
+                "prefix_rows": int(prefix_rows.shape[0]),
+                "new_tokens": int(len(new_tokens)),
+            },
+        )
         # New-token rows: one per step, each (1, H, 1, input_len+i)
         new_rows = [
             step_attn[self.cross_attn_layer][0]
@@ -155,7 +165,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             torch.zeros(0, max(audio_len, 1), device=self.device)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0) # (n_prefix + n_new, audio_len)
-        cross_attn = self.normalize_attn(cross_attn)
 
         return new_tokens, cross_attn
 

From 69eb48f1b1034e41dbe4f38e2b9b94ba386956a0 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:42:44 +0100
Subject: [PATCH 022/157] debug

---
 simulstream/server/speech_processors/base_streamatt.py | 10 +++++-----
 .../server/speech_processors/phi4multimodal_doa.py     |  9 +++++----
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index ab6b7ef..bfb1f78 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -180,11 +180,11 @@ def _update_speech_history(self, discarded_text: int, cross_attn: torch.Tensor)
             "streamatt trim debug",
             {
                 "discarded_text": int(discarded_text),
-                "history_len": int(len(self.text_history)),
-                "cross_attn_rows": int(cross_attn.shape[0]),
-                "cross_attn_cols": int(cross_attn.shape[1]),
-                "most_attended_head": most_attended_idxs[:5].tolist(),
-                "earliest_attended_idx": int(earliest_attended_idx),
+                "retained_history_token_count": int(len(self.text_history)),
+                "retained_text_rows": int(cross_attn.shape[0]),
+                "audio_token_count": int(cross_attn.shape[1]),
+                "peak_audio_positions_sample": most_attended_idxs[:5].tolist(),
+                "earliest_peak_audio_position": int(earliest_attended_idx),
                 "frames_to_cut": int(frames_to_cut),
             },
         )
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 8357d42..c4edc0c 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -148,11 +148,11 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         print(
             "phi4 prefix debug",
             {
-                "history_len": prefix_len,
+                "history_token_count": prefix_len,
                 "input_len": int(input_len),
-                "audio_len": int(audio_len),
-                "prefix_rows": int(prefix_rows.shape[0]),
-                "new_tokens": int(len(new_tokens)),
+                "audio_token_count": int(audio_len),
+                "prefix_text_rows": int(prefix_rows.shape[0]),
+                "new_token_count": int(len(new_tokens)),
             },
         )
         # New-token rows: one per step, each (1, H, 1, input_len+i)
@@ -165,6 +165,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             torch.zeros(0, max(audio_len, 1), device=self.device)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0) # (n_prefix + n_new, audio_len)
+        cross_attn = self.normalize_attn(cross_attn)
 
         return new_tokens, cross_attn
 

From e6f936f250ddd3120ad3112b8564ef7e6145950e Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:46:02 +0100
Subject: [PATCH 023/157] debug

---
 .../server/speech_processors/phi4multimodal_doa.py        | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index c4edc0c..ea6641e 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -161,6 +161,14 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             .mean(dim=0).squeeze(0)[audio_positions]  # (audio_len,)
             for step_attn in output.attentions[1:-1]    # avoid attention of <|end|> token
         ]
+        print(
+            "phi4 attention debug",
+            {
+                "decoded_new_token_count": int(len(new_tokens)),
+                "attention_step_count": int(len(output.attentions)),
+                "new_attention_row_count": int(len(new_rows)),
+            },
+        )
         new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
 

From 616fcaacfbd4180f5a736db2fab63d0693b5f7e6 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:50:51 +0100
Subject: [PATCH 024/157] reduce hallucinations

---
 config/phi4multimodal_doa_frame4.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_frame4.yaml
index 68a73fe..a356079 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_frame4.yaml
@@ -9,3 +9,4 @@ cross_attn_layer: 3
 cutoff_frame_num: 4
 detokenizer_type: "hf"
 word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file

From c629a7635750e378505db15bf20fc3fd11cd7d25 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:51:23 +0100
Subject: [PATCH 025/157] reduce hallucinations

---
 simulstream/server/speech_processors/base_doa.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 8f17b30..49aa69c 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -71,6 +71,7 @@ def __init__(self, config: SimpleNamespace):
         self.src_lang_tag = getattr(self.config, "src_lang_tag", "en")
         self.tgt_lang_tag = getattr(self.config, "tgt_lang_tag", "en")
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.max_new_tokens = getattr(self.config, "max_new_tokens", 128)
 
     @property
     def audio_max_len(self) -> int:

From af083b77b86175908d1d55ceb6e5a744c8fbcd0f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 18:52:06 +0100
Subject: [PATCH 026/157] Try fix

---
 .../server/speech_processors/phi4multimodal_doa.py  | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index ea6641e..2ee3d45 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -155,22 +155,27 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
                 "new_token_count": int(len(new_tokens)),
             },
         )
-        # New-token rows: one per step, each (1, H, 1, input_len+i)
+        # The prefill pass predicts the first generated token, so we use the last prompt row
+        # as its proxy audio-attention. Subsequent generated tokens come from later decode steps.
+        first_new_row = prefill_attn[-1:, audio_positions] if len(new_tokens) > 0 else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
         new_rows = [
             step_attn[self.cross_attn_layer][0]
             .mean(dim=0).squeeze(0)[audio_positions]  # (audio_len,)
-            for step_attn in output.attentions[1:-1]    # avoid attention of <|end|> token
+            for step_attn in output.attentions[1:]
         ]
         print(
             "phi4 attention debug",
             {
                 "decoded_new_token_count": int(len(new_tokens)),
                 "attention_step_count": int(len(output.attentions)),
-                "new_attention_row_count": int(len(new_rows)),
+                "subsequent_new_attention_row_count": int(len(new_rows)),
+                "first_token_proxy_row_count": int(first_new_row.shape[0]),
             },
         )
-        new_attn = torch.stack(new_rows, dim=0) if new_rows else \
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0) # (n_prefix + n_new, audio_len)
         cross_attn = self.normalize_attn(cross_attn)

From 54d137f1a2ce7415a129a041697c41951f2ce4db Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 19:01:38 +0100
Subject: [PATCH 027/157] Debug

---
 .../server/speech_processors/base_streamatt.py     | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index bfb1f78..fe0cc40 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -239,10 +239,24 @@ def alignatt_policy(self, generated_tokens, cross_attn) -> List[str]:
         # Truncate tokens up to the first invalid alignment (if any)
         if len(invalid_tok_ids) > 0:
             selected_tokens = selected_tokens[:invalid_tok_ids[0]]
+        selected_before_word_postprocess = list(selected_tokens)
 
         if self.word_level_postprocess:
             selected_tokens = self._strip_incomplete_words(selected_tokens)
 
+        print(
+            "alignatt selection debug",
+            {
+                "generated_token_count": int(len(generated_tokens)),
+                "audio_token_count": int(cross_attn.size(1)),
+                "cutoff_audio_position": int(cutoff),
+                "peak_audio_positions_sample": most_attended_frames[:8].tolist(),
+                "invalid_token_indices": invalid_tok_ids[:8].tolist(),
+                "selected_count_before_word_postprocess": int(len(selected_before_word_postprocess)),
+                "selected_count_after_word_postprocess": int(len(selected_tokens)),
+            },
+        )
+
         # Store unselected tokens, to be used in the case of end of stream
         self.unselected_tokens = generated_tokens[len(selected_tokens):]
 

From 4d889e341e258e825852228f35c26ac6add908ad Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 19:13:30 +0100
Subject: [PATCH 028/157] Remove unnecessary parameters

---
 config/phi4multimodal_doa_frame4.yaml            | 2 +-
 simulstream/server/speech_processors/base_doa.py | 9 +++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_frame4.yaml
index a356079..f33dbf8 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_frame4.yaml
@@ -5,7 +5,7 @@ text_history:
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 0.5  # seconds
-cross_attn_layer: 3
+attn_layer: 3
 cutoff_frame_num: 4
 detokenizer_type: "hf"
 word_level_postprocess: True  # Disable if character-level language
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 49aa69c..508b068 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -65,13 +65,10 @@ class DecoderOnlyAttention(BaseStreamAtt):
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
-        self.cross_attn_layer = getattr(self.config, "attention_layer", 3)
-        self.max_new_tokens = getattr(self.config, "max_new_tokens", 4096)
-        self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 360)
-        self.src_lang_tag = getattr(self.config, "src_lang_tag", "en")
-        self.tgt_lang_tag = getattr(self.config, "tgt_lang_tag", "en")
+        self.cross_attn_layer = getattr(self.config, "attn_layer", 3)
+        self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.max_new_tokens = getattr(self.config, "max_new_tokens", 128)
+        self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
 
     @property
     def audio_max_len(self) -> int:

From 8fc0ee46aae67650c91de6993c6088aee62762cf Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 19:18:30 +0100
Subject: [PATCH 029/157] Fix stripping with alternative tokenizer and add
 debug

---
 .../server/speech_processors/base_streamatt.py     | 14 ++++++++++++++
 .../server/speech_processors/phi4multimodal_doa.py | 11 ++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index fe0cc40..de6c516 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -15,6 +15,7 @@
 import torch
 import logging
 import numpy as np
+import string
 
 from types import SimpleNamespace
 from abc import abstractmethod
@@ -205,6 +206,19 @@ def _strip_incomplete_words(self, tokens: List[str]) -> List[str]:
         Returns:
             List[str]: A list of generated tokens from which partial words are removed.
         """
+        # Some tokenizers emit a trailing empty token after punctuation/EOS; drop it first so
+        # complete outputs like [" output", ".", ""] are not mistaken for incomplete words
+        while tokens and tokens[-1] == "":
+            tokens = tokens[:-1]
+
+        if not tokens:
+            return []
+
+        last_token = tokens[-1].strip()
+        # If the hypothesis already ends with punctuation, keep it as a complete segment
+        if last_token and last_token[-1] in string.punctuation:
+            return tokens
+
         tokens_to_write = []
         # iterate from the end and count how many trailing tokens to drop
         num_tokens_incomplete = 0
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 2ee3d45..c3dbee7 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -78,11 +78,20 @@ def build_prompt(self) -> str:
             .replace("{src_lang}", LANG_MAPPER[self.src_lang])
             .replace("{tgt_lang}", LANG_MAPPER[self.tgt_lang]))
         prefix = "".join(self.text_history) if self.text_history else ""
-        return (
+        prompt = (
             f"{self._USER_START}{self._AUDIO_TOKEN}"
             f"{filled_prompt}{self._END_TOKEN}"
             f"{self._ASST_START}{prefix}"
         )
+        print(
+            "phi4 prompt debug",
+            {
+                "src_lang": self.src_lang,
+                "tgt_lang": self.tgt_lang,
+                "prompt": prompt,
+            },
+        )
+        return prompt
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         return self.processor(

From 6a40aedba0cc16427722e7fa38e6ef2d7a843194 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 19:22:09 +0100
Subject: [PATCH 030/157] Choose a simple prompt for Phi4Multimodal

---
 .../speech_processors/phi4multimodal_doa.py       | 15 ++-------------
 1 file changed, 2 insertions(+), 13 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index c3dbee7..636a8d3 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -21,7 +21,7 @@
 from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
 
 from simulstream.server.speech_processors import SAMPLE_RATE, class_load
-from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, TEMPLATED_SPEECH_PROMPT, LANG_MAPPER
+from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, LANG_MAPPER
 
 
 class Phi4MultimodalDOA(DecoderOnlyAttention):
@@ -73,24 +73,13 @@ def audio_max_len(self) -> int:
         return getattr(self.config, "audio_max_frames", 480_000)
 
     def build_prompt(self) -> str:
-        filled_prompt = (
-            TEMPLATED_SPEECH_PROMPT
-            .replace("{src_lang}", LANG_MAPPER[self.src_lang])
-            .replace("{tgt_lang}", LANG_MAPPER[self.tgt_lang]))
+        filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
         prefix = "".join(self.text_history) if self.text_history else ""
         prompt = (
             f"{self._USER_START}{self._AUDIO_TOKEN}"
             f"{filled_prompt}{self._END_TOKEN}"
             f"{self._ASST_START}{prefix}"
         )
-        print(
-            "phi4 prompt debug",
-            {
-                "src_lang": self.src_lang,
-                "tgt_lang": self.tgt_lang,
-                "prompt": prompt,
-            },
-        )
         return prompt
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:

From b215ba3912a34275f4bd1da22b49239c0fa0cdef Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 24 Mar 2026 19:24:06 +0100
Subject: [PATCH 031/157] More stable outputs

---
 config/phi4multimodal_doa_frame4.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_frame4.yaml
index f33dbf8..b39573c 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_frame4.yaml
@@ -4,7 +4,7 @@ text_history:
   history_words: 10
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
-speech_chunk_size: 0.5  # seconds
+speech_chunk_size: 1  # seconds
 attn_layer: 3
 cutoff_frame_num: 4
 detokenizer_type: "hf"

From 2c888314ad7c2b0df7ab7deade93cee0382e6049 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 27 Mar 2026 11:15:04 +0100
Subject: [PATCH 032/157] Remove debugs

---
 .../speech_processors/base_streamatt.py       | 32 -------------------
 .../speech_processors/phi4multimodal_doa.py   | 19 -----------
 2 files changed, 51 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index de6c516..83ce7f8 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -177,18 +177,6 @@ def _update_speech_history(self, discarded_text: int, cross_attn: torch.Tensor)
 
         # Multiply by the subsampling factor to recover the original number of frames
         frames_to_cut = earliest_attended_idx * self.audio_subsampling_factor
-        print(
-            "streamatt trim debug",
-            {
-                "discarded_text": int(discarded_text),
-                "retained_history_token_count": int(len(self.text_history)),
-                "retained_text_rows": int(cross_attn.shape[0]),
-                "audio_token_count": int(cross_attn.shape[1]),
-                "peak_audio_positions_sample": most_attended_idxs[:5].tolist(),
-                "earliest_peak_audio_position": int(earliest_attended_idx),
-                "frames_to_cut": int(frames_to_cut),
-            },
-        )
 
         # Cut the unattended audio features
         self.audio_history = self.audio_history[frames_to_cut:]
@@ -253,24 +241,10 @@ def alignatt_policy(self, generated_tokens, cross_attn) -> List[str]:
         # Truncate tokens up to the first invalid alignment (if any)
         if len(invalid_tok_ids) > 0:
             selected_tokens = selected_tokens[:invalid_tok_ids[0]]
-        selected_before_word_postprocess = list(selected_tokens)
 
         if self.word_level_postprocess:
             selected_tokens = self._strip_incomplete_words(selected_tokens)
 
-        print(
-            "alignatt selection debug",
-            {
-                "generated_token_count": int(len(generated_tokens)),
-                "audio_token_count": int(cross_attn.size(1)),
-                "cutoff_audio_position": int(cutoff),
-                "peak_audio_positions_sample": most_attended_frames[:8].tolist(),
-                "invalid_token_indices": invalid_tok_ids[:8].tolist(),
-                "selected_count_before_word_postprocess": int(len(selected_before_word_postprocess)),
-                "selected_count_after_word_postprocess": int(len(selected_tokens)),
-            },
-        )
-
         # Store unselected tokens, to be used in the case of end of stream
         self.unselected_tokens = generated_tokens[len(selected_tokens):]
 
@@ -288,19 +262,13 @@ def process_chunk(self, waveform: np.float32) -> IncrementalOutput:
         speech = self._preprocess(waveform)
         # Generate new hypothesis with its corresponding cross-attention scores (no prefix)
         generated_tokens, cross_attn = self._generate(speech)
-        print(generated_tokens, cross_attn.shape)
         # Select the part of the new hypothesis to be emitted, and trim cross-attention accordingly
         selected_output = self.alignatt_policy(generated_tokens, cross_attn)
-        print(f"selected {selected_output}")
         incremental_output = self._build_incremental_outputs(selected_output)
         # Discard textual history, if needed
-        print(f"text history {self.text_history}")
         discarded_text = self._update_text_history(selected_output)
-        print(f"discarded {discarded_text}")
         # Trim audio corresponding to the discarded textual history
-        print(f"previous speech history {self.audio_history.shape}")
         self._update_speech_history(discarded_text, cross_attn)
-        print(f"trimmed speech history {self.audio_history.shape}")
         return incremental_output
 
     def end_of_stream(self) -> IncrementalOutput:
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 636a8d3..76d2aac 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -143,16 +143,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
         else:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
-        print(
-            "phi4 prefix debug",
-            {
-                "history_token_count": prefix_len,
-                "input_len": int(input_len),
-                "audio_token_count": int(audio_len),
-                "prefix_text_rows": int(prefix_rows.shape[0]),
-                "new_token_count": int(len(new_tokens)),
-            },
-        )
         # The prefill pass predicts the first generated token, so we use the last prompt row
         # as its proxy audio-attention. Subsequent generated tokens come from later decode steps.
         first_new_row = prefill_attn[-1:, audio_positions] if len(new_tokens) > 0 else \
@@ -162,15 +152,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             .mean(dim=0).squeeze(0)[audio_positions]  # (audio_len,)
             for step_attn in output.attentions[1:]
         ]
-        print(
-            "phi4 attention debug",
-            {
-                "decoded_new_token_count": int(len(new_tokens)),
-                "attention_step_count": int(len(output.attentions)),
-                "subsequent_new_attention_row_count": int(len(new_rows)),
-                "first_token_proxy_row_count": int(first_new_row.shape[0]),
-            },
-        )
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)

From a4bc581557f85d557d219343b14ec319e808f934 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 3 Apr 2026 16:45:45 +0200
Subject: [PATCH 033/157] Add hf_model_name to config of DOA Phi

---
 config/phi4multimodal_doa_frame4.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_frame4.yaml
index b39573c..ee51a96 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_frame4.yaml
@@ -8,5 +8,6 @@ speech_chunk_size: 1  # seconds
 attn_layer: 3
 cutoff_frame_num: 4
 detokenizer_type: "hf"
+hf_model_name: "microsoft/Phi-4-multimodal-instruct"
 word_level_postprocess: True  # Disable if character-level language
 max_new_tokens: 32
\ No newline at end of file

From 031482e8186f3e6804020b5177c3e237280b5cd8 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 3 Apr 2026 16:52:32 +0200
Subject: [PATCH 034/157] Add trust_remote True to bypass manually consent
 during eval

---
 simulstream/metrics/detokenizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 72fbad9..57b234a 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -21,7 +21,7 @@ def build_hf_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
 
     assert hasattr(config, "hf_model_name"), \
         "`hf_model_name` required in the eval config for `hf` detokenizer"
-    processor = AutoProcessor.from_pretrained(config.hf_model_name)
+    processor = AutoProcessor.from_pretrained(config.hf_model_name, trust_ret_code=True)
 
     def detokenize(input_tokens: List[str]) -> str:
         return processor.tokenizer.convert_tokens_to_string(input_tokens)

From c065466d907ab8b4566f986b2d762780351df243 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 3 Apr 2026 16:53:42 +0200
Subject: [PATCH 035/157] Revert the punctuation stripping

---
 .../server/speech_processors/base_streamatt.py      | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 83ce7f8..db41803 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -194,19 +194,6 @@ def _strip_incomplete_words(self, tokens: List[str]) -> List[str]:
         Returns:
             List[str]: A list of generated tokens from which partial words are removed.
         """
-        # Some tokenizers emit a trailing empty token after punctuation/EOS; drop it first so
-        # complete outputs like [" output", ".", ""] are not mistaken for incomplete words
-        while tokens and tokens[-1] == "":
-            tokens = tokens[:-1]
-
-        if not tokens:
-            return []
-
-        last_token = tokens[-1].strip()
-        # If the hypothesis already ends with punctuation, keep it as a complete segment
-        if last_token and last_token[-1] in string.punctuation:
-            return tokens
-
         tokens_to_write = []
         # iterate from the end and count how many trailing tokens to drop
         num_tokens_incomplete = 0

From 0b95426189db77f2cdabccef5fcb15c4d567956e Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 3 Apr 2026 18:31:43 +0200
Subject: [PATCH 036/157] Improve DOA descriptions and remove unnecessary
 functions

---
 .../server/speech_processors/base_doa.py      | 19 +++++++++----------
 .../speech_processors/phi4multimodal_doa.py   | 11 -----------
 2 files changed, 9 insertions(+), 21 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 508b068..13e671e 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -42,10 +42,10 @@ class DecoderOnlyAttention(BaseStreamAtt):
     Generic Decoder-only Attention-based policy for SpeechLLMs.
 
     The class handles:
-    - Rolling raw-waveform history accumulation.
+    - Raw-waveform history accumulation.
     - Greedy generation with ``output_attentions=True``.
     - Building the proxy cross-attention matrix from self-attention weights.
-    - Token decoding.
+    - Applying StreamAtt-based policy on the proxy cross-attention matrix.
 
     Subclasses must implement the five abstract methods listed below.
 
@@ -53,19 +53,18 @@ class DecoderOnlyAttention(BaseStreamAtt):
     ----------
     config : SimpleNamespace
         All fields from :class:`BaseStreamAtt`, plus:
-
-        device : str
-            Torch device string.  Default: ``"cuda"``.
-        audio_max_frames : int
-            Maximum raw waveform samples to keep in the rolling history
-            (at 16 kHz).  Default: ``480_000`` (30 s).
+        attn_layer : int
+            Layer from which to extract attention scores. Default: ``0``.
+        audio_history_max_duration : int
+            Maximum raw waveform length to keep in the rolling history.
+            Default: ``180`` (seconds).
         max_new_tokens : int
-            Maximum tokens to generate per chunk.  Default: ``200``.
+            Maximum tokens to generate per chunk.  Default: ``32``.
     """
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
-        self.cross_attn_layer = getattr(self.config, "attn_layer", 3)
+        self.cross_attn_layer = getattr(self.config, "attn_layer", 0)
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 76d2aac..effbf06 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -27,13 +27,6 @@
 class Phi4MultimodalDOA(DecoderOnlyAttention):
     """
     Decoder-Only Attention agent for ``microsoft/Phi-4-multimodal-instruct``.
-
-    Extra config fields
-    -------------------
-    model_path : str
-        Default: ``"microsoft/Phi-4-multimodal-instruct"``
-    target_lang : str
-        Target language when ``task="translate"``.  Default: ``"English"``
     """
 
     # Phi-4 special tokens
@@ -68,10 +61,6 @@ def load_model(cls, config: SimpleNamespace) -> None:
         cls.model.eval()
         cls.generation_config = GenerationConfig.from_pretrained(model_path)
 
-    @property
-    def audio_max_len(self) -> int:
-        return getattr(self.config, "audio_max_frames", 480_000)
-
     def build_prompt(self) -> str:
         filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
         prefix = "".join(self.text_history) if self.text_history else ""

From 634759994aace9f7b60c79e80ae50da8b5c8c14f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 3 Apr 2026 18:31:54 +0200
Subject: [PATCH 037/157] Remove unnecessary import

---
 simulstream/server/speech_processors/base_streamatt.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index db41803..249c76a 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -15,7 +15,6 @@
 import torch
 import logging
 import numpy as np
-import string
 
 from types import SimpleNamespace
 from abc import abstractmethod

From 587655256b3e948e28663864f1da3cc86e9dee0a Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 8 Apr 2026 09:16:59 +0200
Subject: [PATCH 038/157] Update language maps

---
 simulstream/server/speech_processors/base_doa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 13e671e..0eb0e70 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -34,7 +34,7 @@
      "translation, without any additional explanations or commentary. Please translate the "
      "provided {src_lang} speech into {tgt_lang}:")
 
-LANG_MAPPER = {"en": "English", "it": "Italian"}
+LANG_MAPPER = {"en": "English", "it": "Italian", "de": "German", "zh": "Chinese (simplified)"}
 
 
 class DecoderOnlyAttention(BaseStreamAtt):

From e3e41c29b0cf67b03e2f4cd67554b45f7457d006 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 8 Apr 2026 09:27:29 +0200
Subject: [PATCH 039/157] Update qwen2.5omni implementation

---
 config/qwen2.5omni_doa_frame4_fixedwords.yaml |  14 ++
 .../server/speech_processors/qwen2_5_doa.py   | 170 ++++++++++++++++++
 2 files changed, 184 insertions(+)
 create mode 100644 config/qwen2.5omni_doa_frame4_fixedwords.yaml
 create mode 100644 simulstream/server/speech_processors/qwen2_5_doa.py

diff --git a/config/qwen2.5omni_doa_frame4_fixedwords.yaml b/config/qwen2.5omni_doa_frame4_fixedwords.yaml
new file mode 100644
index 0000000..5df6858
--- /dev/null
+++ b/config/qwen2.5omni_doa_frame4_fixedwords.yaml
@@ -0,0 +1,14 @@
+type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2OmniDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
+  history_words: 10
+audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: 3
+cutoff_frame_num: 4
+detokenizer_type: "hf"
+hf_model_name: "Qwen/Qwen2.5-Omni-3B"
+use_video: True
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
new file mode 100644
index 0000000..e32497a
--- /dev/null
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -0,0 +1,170 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import logging
+from types import SimpleNamespace
+from typing import List, Tuple
+
+import numpy as np
+import torch
+
+from qwen_omni_utils import process_mm_info
+from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
+
+from simulstream.server.speech_processors import SAMPLE_RATE, class_load
+from simulstream.server.speech_processors.base_doa import (
+    DecoderOnlyAttention,
+    LANG_MAPPER,
+    TEMPLATED_SPEECH_PROMPT,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Qwen2OmniDOA(DecoderOnlyAttention):
+    """
+    Decoder-Only Attention agent for ``Qwen/Qwen2.5-Omni-*``.
+
+    Extra config fields
+    -------------------
+    hf_model_name : str
+        Default: ``"Qwen/Qwen2.5-Omni-7B"``.
+        ``"Qwen/Qwen2.5-Omni-3B"`` is also supported.
+    """
+
+    BOW_PREFIX = " "
+    AUDIO_TOKEN_STRIDE = 640
+    SYSTEM_PROMPT = (
+        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
+        "perceiving auditory and visual inputs, as well as generating text and speech. Only "
+        "return the answer requested. Do not include any explanation or introductions."
+    )
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.bow_prefix = self.BOW_PREFIX
+        text_history_cls = class_load(self.text_history_config.type)
+        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
+        self.use_video =  getattr(self.config, "use_video", False)
+
+    @classmethod
+    def load_model(cls, config: SimpleNamespace) -> None:
+        model_name = getattr(
+            config,
+            "hf_model_name",
+            getattr(config, "model_path", "Qwen/Qwen2.5-Omni-7B"),
+        )
+        attn_impl = getattr(config, "attn_implementation", "flash_attention_2")
+
+        cls.model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
+            model_name,
+            torch_dtype="auto",
+            device_map="auto",
+            attn_implementation=attn_impl,
+        )
+        cls.processor = Qwen2_5OmniProcessor.from_pretrained(model_name)
+        cls.model.eval()
+
+    def build_prompt(self) -> str:
+        return (
+            TEMPLATED_SPEECH_PROMPT
+            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
+            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
+        )
+
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        conversation = [
+            {
+                "role": "system",
+                "content": [{"type": "text", "text": self.SYSTEM_PROMPT}],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "audio", "audio": waveform},
+                    {"type": "text", "text": self.build_prompt()},
+                ],
+            },
+        ]
+
+        prompt = self.processor.apply_chat_template(
+            conversation,
+            add_generation_prompt=True,
+            tokenize=False,
+        )
+        prefix = "".join(self.text_history) if self.text_history else ""
+        audios, images, videos = process_mm_info(conversation, use_audio_in_video=False)
+
+        return self.processor(
+            text=f"{prompt}{prefix}",
+            audio=audios,
+            images=images,
+            videos=videos,
+            sampling_rate=SAMPLE_RATE,
+            return_tensors="pt",
+            padding=True,
+            use_audio_in_video=True,
+        ).to(self.device)
+
+    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+        input_ids = inputs["input_ids"]
+        input_len = input_ids.shape[1]
+
+        audio_token_id = getattr(self.model.config, "audio_token_index", None)
+        if audio_token_id is None:
+            raise ValueError("Qwen2.5-Omni config is missing `audio_token_index`.")
+        audio_positions = (input_ids[0] == audio_token_id).nonzero(as_tuple=True)[0]
+        audio_len = audio_positions.shape[0]
+
+        output = self.model.generate(
+            **inputs,
+            generation_mode="text",
+            use_audio_in_video=False,
+            thinker_max_new_tokens=self.max_new_tokens,
+            thinker_output_attentions=True,
+            thinker_return_dict_in_generate=True,
+            thinker_do_sample=False,
+        )
+
+        new_ids = output.sequences[:, input_len:]
+        new_tokens = [
+            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in new_ids[0]
+        ]
+
+        prefill_attn = output.attentions[0][self.cross_attn_layer][0].mean(dim=0)
+        prefix_len = len(self.text_history) if self.text_history else 0
+        if prefix_len > 0:
+            prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
+        else:
+            prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
+
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
+        new_rows = [
+            step_attn[self.cross_attn_layer][0].mean(dim=0).squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:]
+        ]
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
+        cross_attn = self.normalize_attn(cross_attn)
+        return new_tokens, cross_attn
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens).strip()

From 1f7f80b3b8c59b85c762fa3bfa52ac9570e1891b Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 11:03:47 +0200
Subject: [PATCH 040/157] Correct trust_remote_code typo in detokenizers

---
 simulstream/metrics/detokenizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 57b234a..50cf7b5 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -21,7 +21,7 @@ def build_hf_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
 
     assert hasattr(config, "hf_model_name"), \
         "`hf_model_name` required in the eval config for `hf` detokenizer"
-    processor = AutoProcessor.from_pretrained(config.hf_model_name, trust_ret_code=True)
+    processor = AutoProcessor.from_pretrained(config.hf_model_name, trust_remote_code=True)
 
     def detokenize(input_tokens: List[str]) -> str:
         return processor.tokenizer.convert_tokens_to_string(input_tokens)

From 9a4314bb0676af776fe4819c7407a39c3e45c435 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 11:18:53 +0200
Subject: [PATCH 041/157] Update config

---
 config/qwen2.5omni_doa_frame4_fixedwords.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/qwen2.5omni_doa_frame4_fixedwords.yaml b/config/qwen2.5omni_doa_frame4_fixedwords.yaml
index 5df6858..99101cb 100644
--- a/config/qwen2.5omni_doa_frame4_fixedwords.yaml
+++ b/config/qwen2.5omni_doa_frame4_fixedwords.yaml
@@ -5,7 +5,7 @@ text_history:
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
-attn_layer: 3
+attn_layer: __LAYER__
 cutoff_frame_num: 4
 detokenizer_type: "hf"
 hf_model_name: "Qwen/Qwen2.5-Omni-3B"

From 5e8c6e313d1ac0dbee8f5a31d866cfc46fe384b4 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 11:29:41 +0200
Subject: [PATCH 042/157] Update Qwen name

---
 config/qwen2.5omni_doa_frame4_fixedwords.yaml       | 2 +-
 simulstream/server/speech_processors/qwen2_5_doa.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/qwen2.5omni_doa_frame4_fixedwords.yaml b/config/qwen2.5omni_doa_frame4_fixedwords.yaml
index 99101cb..0e2916e 100644
--- a/config/qwen2.5omni_doa_frame4_fixedwords.yaml
+++ b/config/qwen2.5omni_doa_frame4_fixedwords.yaml
@@ -1,4 +1,4 @@
-type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2OmniDOA"
+type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
   history_words: 10
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index e32497a..1258840 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -33,7 +33,7 @@
 logger = logging.getLogger(__name__)
 
 
-class Qwen2OmniDOA(DecoderOnlyAttention):
+class Qwen2_5OmniDOA(DecoderOnlyAttention):
     """
     Decoder-Only Attention agent for ``Qwen/Qwen2.5-Omni-*``.
 

From bdd3253ce568f387e1c900a2831b2686105425ba Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 11:56:31 +0200
Subject: [PATCH 043/157] Update Qwen code

---
 .../server/speech_processors/qwen2_5_doa.py   | 36 +++++++++++++++----
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 1258840..dc92613 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -46,6 +46,9 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
 
     BOW_PREFIX = " "
     AUDIO_TOKEN_STRIDE = 640
+    AUDIO_TOKEN_INDEX = 151646
+    AUDIO_START_TOKEN_ID = 151647
+    AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
         "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
         "perceiving auditory and visual inputs, as well as generating text and speech. Only "
@@ -106,7 +109,7 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             tokenize=False,
         )
         prefix = "".join(self.text_history) if self.text_history else ""
-        audios, images, videos = process_mm_info(conversation, use_audio_in_video=False)
+        audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
 
         return self.processor(
             text=f"{prompt}{prefix}",
@@ -119,20 +122,41 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             use_audio_in_video=True,
         ).to(self.device)
 
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+        if audio_positions.numel() > 0:
+            return audio_positions
+
+        start_positions = (input_ids[0] == self.AUDIO_START_TOKEN_ID).nonzero(as_tuple=True)[0]
+        end_positions = (input_ids[0] == self.AUDIO_END_TOKEN_ID).nonzero(as_tuple=True)[0]
+        if start_positions.numel() == 0 or end_positions.numel() == 0:
+            raise ValueError(
+                "Qwen2.5-Omni audio tokens were not found in the prompt. Checked "
+                "`audio_token_index`, `<|audio_bos|>`, and `<|audio_eos|>`."
+            )
+
+        start_pos = start_positions[0]
+        end_positions = end_positions[end_positions > start_pos]
+        if end_positions.numel() == 0:
+            raise ValueError("Qwen2.5-Omni found `<|audio_bos|>` but not a matching `<|audio_eos|>`.")
+
+        end_pos = end_positions[0]
+        if end_pos <= start_pos + 1:
+            raise ValueError("Qwen2.5-Omni found empty audio span between `<|audio_bos|>` and `<|audio_eos|>`.")
+
+        return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
+
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         input_ids = inputs["input_ids"]
         input_len = input_ids.shape[1]
 
-        audio_token_id = getattr(self.model.config, "audio_token_index", None)
-        if audio_token_id is None:
-            raise ValueError("Qwen2.5-Omni config is missing `audio_token_index`.")
-        audio_positions = (input_ids[0] == audio_token_id).nonzero(as_tuple=True)[0]
+        audio_positions = self._find_audio_positions(input_ids)
         audio_len = audio_positions.shape[0]
 
         output = self.model.generate(
             **inputs,
             generation_mode="text",
-            use_audio_in_video=False,
+            use_audio_in_video=True,
             thinker_max_new_tokens=self.max_new_tokens,
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,

From cef90d6d91e5dc0504b47285a5ad4beb43bc83f6 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 11:59:56 +0200
Subject: [PATCH 044/157] Update Qwen code

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index dc92613..a4fbf58 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -155,13 +155,15 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         output = self.model.generate(
             **inputs,
-            generation_mode="text",
             use_audio_in_video=True,
+            return_audio=False,
             thinker_max_new_tokens=self.max_new_tokens,
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
         )
+        if isinstance(output, tuple):
+            output = output[0]
 
         new_ids = output.sequences[:, input_len:]
         new_tokens = [

From 4010d6e9e764c89b736b3fbf44c076a7cafa1982 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 12:01:05 +0200
Subject: [PATCH 045/157] Disable flash attention for testing

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index a4fbf58..92992e9 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -70,7 +70,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
             "hf_model_name",
             getattr(config, "model_path", "Qwen/Qwen2.5-Omni-7B"),
         )
-        attn_impl = getattr(config, "attn_implementation", "flash_attention_2")
+        attn_impl = getattr(config, "attn_implementation", "eager") #"flash_attention_2")
 
         cls.model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
             model_name,

From 4ff9daea0381c62b375f83d415121959f31183a4 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 12:04:20 +0200
Subject: [PATCH 046/157] Revert to standard system prompt

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 92992e9..41bb935 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -51,8 +51,7 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
         "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
-        "perceiving auditory and visual inputs, as well as generating text and speech. Only "
-        "return the answer requested. Do not include any explanation or introductions."
+        "perceiving auditory and visual inputs, as well as generating text and speech."
     )
 
     def __init__(self, config: SimpleNamespace):

From faf060abc2dcf4365e8117746c01aa8dfe2e174a Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 12:10:25 +0200
Subject: [PATCH 047/157] Add average across layers

---
 simulstream/server/speech_processors/base_doa.py    | 13 +++++++++++++
 .../server/speech_processors/phi4multimodal_doa.py  |  9 +++++----
 simulstream/server/speech_processors/qwen2_5_doa.py |  4 ++--
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 0eb0e70..0f80055 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -55,6 +55,10 @@ class DecoderOnlyAttention(BaseStreamAtt):
         All fields from :class:`BaseStreamAtt`, plus:
         attn_layer : int
             Layer from which to extract attention scores. Default: ``0``.
+        average_attn_over_layers : bool
+            Whether to average attention over all decoder layers instead of
+            using the single layer selected by ``attn_layer``.
+            Default: ``False``.
         audio_history_max_duration : int
             Maximum raw waveform length to keep in the rolling history.
             Default: ``180`` (seconds).
@@ -65,6 +69,7 @@ class DecoderOnlyAttention(BaseStreamAtt):
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
         self.cross_attn_layer = getattr(self.config, "attn_layer", 0)
+        self.average_attn_over_layers = getattr(self.config, "average_attn_over_layers", False)
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
@@ -128,6 +133,14 @@ def set_target_language(self, language: str) -> None:
     def set_source_language(self, language: str) -> None:
         self.src_lang = language
 
+    def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
+        if self.average_attn_over_layers:
+            return torch.stack(
+                [layer_attn[0].mean(dim=0) for layer_attn in step_attn],
+                dim=0,
+            ).mean(dim=0)
+        return step_attn[self.cross_attn_layer][0].mean(dim=0)
+
     def _preprocess(self, waveform: np.float32) -> dict:
         """
         Append *waveform* to ``self.audio_history``, enforce the maximum length,
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index effbf06..4b0a7b3 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -125,8 +125,9 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         # Build proxy cross-attention for the hypothesis (prefix + new_tokens) ────────────────────
         # Prefix rows from the prefill pass
         # output.attentions[0][layer]: (1, H, input_len, input_len)
-        prefill_attn = (output.attentions[0][self.cross_attn_layer][0]
-                        .mean(dim=0))  # (input_len, input_len)
+        prefill_attn = self.mean_attn_over_heads_and_selected_layers(
+            output.attentions[0]
+        )  # (input_len, input_len)
         prefix_len = len(self.text_history) if self.text_history else 0
         if prefix_len > 0:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
@@ -137,8 +138,8 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         first_new_row = prefill_attn[-1:, audio_positions] if len(new_tokens) > 0 else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_rows = [
-            step_attn[self.cross_attn_layer][0]
-            .mean(dim=0).squeeze(0)[audio_positions]  # (audio_len,)
+            self.mean_attn_over_heads_and_selected_layers(step_attn)
+            .squeeze(0)[audio_positions]  # (audio_len,)
             for step_attn in output.attentions[1:]
         ]
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 41bb935..027d752 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -170,7 +170,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             for token_id in new_ids[0]
         ]
 
-        prefill_attn = output.attentions[0][self.cross_attn_layer][0].mean(dim=0)
+        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
         if prefix_len > 0:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
@@ -180,7 +180,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_rows = [
-            step_attn[self.cross_attn_layer][0].mean(dim=0).squeeze(0)[audio_positions]
+            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]
         ]
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \

From 1f7c34fe5a38c09eb20425c7a5992af03bbdeaa1 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 12:14:57 +0200
Subject: [PATCH 048/157] Update config

---
 ...rds.yaml => qwen2.5omni_3b_doa_fixedwords.yaml} |  7 ++++---
 config/qwen2.5omni_3b_doa_punctuation.yaml         | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)
 rename config/{qwen2.5omni_doa_frame4_fixedwords.yaml => qwen2.5omni_3b_doa_fixedwords.yaml} (75%)
 create mode 100644 config/qwen2.5omni_3b_doa_punctuation.yaml

diff --git a/config/qwen2.5omni_doa_frame4_fixedwords.yaml b/config/qwen2.5omni_3b_doa_fixedwords.yaml
similarity index 75%
rename from config/qwen2.5omni_doa_frame4_fixedwords.yaml
rename to config/qwen2.5omni_3b_doa_fixedwords.yaml
index 0e2916e..11d7385 100644
--- a/config/qwen2.5omni_doa_frame4_fixedwords.yaml
+++ b/config/qwen2.5omni_3b_doa_fixedwords.yaml
@@ -2,13 +2,14 @@ type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
   history_words: 10
-audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
+audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__
-cutoff_frame_num: 4
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
 hf_model_name: "Qwen/Qwen2.5-Omni-3B"
-use_video: True
+use_video: False
 word_level_postprocess: True  # Disable if character-level language
 max_new_tokens: 32
\ No newline at end of file
diff --git a/config/qwen2.5omni_3b_doa_punctuation.yaml b/config/qwen2.5omni_3b_doa_punctuation.yaml
new file mode 100644
index 0000000..aeaaa58
--- /dev/null
+++ b/config/qwen2.5omni_3b_doa_punctuation.yaml
@@ -0,0 +1,14 @@
+type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
+detokenizer_type: "hf"
+hf_model_name: "Qwen/Qwen2.5-Omni-3B"
+use_video: False
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file

From 53e333855860090deca94d825b73bb7477ad07eb Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 14:02:20 +0200
Subject: [PATCH 049/157] Add Phi4Multimodal configs

---
 config/phi4multimodal_doa_fixedwords.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 config/phi4multimodal_doa_fixedwords.yaml

diff --git a/config/phi4multimodal_doa_fixedwords.yaml b/config/phi4multimodal_doa_fixedwords.yaml
new file mode 100644
index 0000000..bf3a82a
--- /dev/null
+++ b/config/phi4multimodal_doa_fixedwords.yaml
@@ -0,0 +1,14 @@
+type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
+  history_words: 10
+audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+cutoff_frame_num: __FRAME__
+average_attn_over_layers: True
+detokenizer_type: "hf"
+hf_model_name: "microsoft/Phi-4-multimodal-instruct"
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file

From a7b6982d326a96f146d9f966b8569cff862b6d40 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 14:02:56 +0200
Subject: [PATCH 050/157] Add Phi4Multimodal configs

---
 config/phi4multimodal_doa_punctuation.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 config/phi4multimodal_doa_punctuation.yaml

diff --git a/config/phi4multimodal_doa_punctuation.yaml b/config/phi4multimodal_doa_punctuation.yaml
new file mode 100644
index 0000000..bb22119
--- /dev/null
+++ b/config/phi4multimodal_doa_punctuation.yaml
@@ -0,0 +1,13 @@
+type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+cutoff_frame_num: __FRAME__
+average_attn_over_layers: True
+detokenizer_type: "hf"
+hf_model_name: "microsoft/Phi-4-multimodal-instruct"
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file

From f3d67133ca79ed964e35c7c966e1603ecec9b61c Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 16:29:13 +0200
Subject: [PATCH 051/157] Fix detokenizer for Qwen2.5-Omni

---
 simulstream/metrics/detokenizers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 50cf7b5..defdadd 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -22,9 +22,10 @@ def build_hf_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
     assert hasattr(config, "hf_model_name"), \
         "`hf_model_name` required in the eval config for `hf` detokenizer"
     processor = AutoProcessor.from_pretrained(config.hf_model_name, trust_remote_code=True)
+    tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
 
     def detokenize(input_tokens: List[str]) -> str:
-        return processor.tokenizer.convert_tokens_to_string(input_tokens)
+        return tokenizer.convert_tokens_to_string(input_tokens)
 
     return detokenize
 

From d9e0ac511d79b3d4b30cf2626cf43237afb67f06 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 18:05:19 +0200
Subject: [PATCH 052/157] Discourage Qwen2.5 Omni to repeat

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 027d752..c76afa3 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -42,6 +42,10 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     hf_model_name : str
         Default: ``"Qwen/Qwen2.5-Omni-7B"``.
         ``"Qwen/Qwen2.5-Omni-3B"`` is also supported.
+    repetition_penalty : float
+        Repetition penalty for text generation. Default: ``1.0``.
+    no_repeat_ngram_size : int
+        N-gram blocking size for text generation. Default: ``0``.
     """
 
     BOW_PREFIX = " "
@@ -61,6 +65,8 @@ def __init__(self, config: SimpleNamespace):
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
         self.use_video =  getattr(self.config, "use_video", False)
+        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.1)
+        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 4)
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
@@ -157,6 +163,8 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             use_audio_in_video=True,
             return_audio=False,
             thinker_max_new_tokens=self.max_new_tokens,
+            thinker_repetition_penalty=self.repetition_penalty,
+            thinker_no_repeat_ngram_size=self.no_repeat_ngram_size,
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,

From 470573347dbe3b99de60fdc22ec853ecee0e19f4 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 21:17:08 +0200
Subject: [PATCH 053/157] Adjust repetition

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index c76afa3..294604f 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -65,8 +65,8 @@ def __init__(self, config: SimpleNamespace):
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
         self.use_video =  getattr(self.config, "use_video", False)
-        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.1)
-        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 4)
+        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.05)
+        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 5)
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:

From 3dae1a5cbea127b9c0ab64a1d4531ffa73343b24 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 9 Apr 2026 21:19:32 +0200
Subject: [PATCH 054/157] Add history param

---
 config/phi4multimodal_doa_fixedwords.yaml | 2 +-
 config/qwen2.5omni_3b_doa_fixedwords.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/phi4multimodal_doa_fixedwords.yaml b/config/phi4multimodal_doa_fixedwords.yaml
index bf3a82a..3a2cb83 100644
--- a/config/phi4multimodal_doa_fixedwords.yaml
+++ b/config/phi4multimodal_doa_fixedwords.yaml
@@ -1,7 +1,7 @@
 type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
-  history_words: 10
+  history_words: __HISTORY__
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
diff --git a/config/qwen2.5omni_3b_doa_fixedwords.yaml b/config/qwen2.5omni_3b_doa_fixedwords.yaml
index 11d7385..621b071 100644
--- a/config/qwen2.5omni_3b_doa_fixedwords.yaml
+++ b/config/qwen2.5omni_3b_doa_fixedwords.yaml
@@ -1,7 +1,7 @@
 type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
-  history_words: 10
+  history_words: __HISTORY__
 audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds

From b384d24fa6bf1cb8ae1091761c8d87de434135ed Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 10 Apr 2026 11:09:11 +0200
Subject: [PATCH 055/157] Add deterministic params

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 5 +++++
 simulstream/server/speech_processors/qwen2_5_doa.py        | 6 ++++++
 2 files changed, 11 insertions(+)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 4b0a7b3..cffb0e9 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -23,6 +23,10 @@
 from simulstream.server.speech_processors import SAMPLE_RATE, class_load
 from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, LANG_MAPPER
 
+from transformers import set_seed
+torch.manual_seed(42)
+set_seed(42)
+
 
 class Phi4MultimodalDOA(DecoderOnlyAttention):
     """
@@ -113,6 +117,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output_attentions=True,
             return_dict_in_generate=True,
             do_sample=False,
+            temperature=0.0,
         )
 
         # Decode newly generated tokens only ──────────────────────────────────────────────────────
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 294604f..5fd61a6 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -29,6 +29,10 @@
     TEMPLATED_SPEECH_PROMPT,
 )
 
+from transformers import set_seed
+torch.manual_seed(42)
+set_seed(42)
+
 
 logger = logging.getLogger(__name__)
 
@@ -168,6 +172,8 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
+            do_sample=False,
+            temperature=0.0,
         )
         if isinstance(output, tuple):
             output = output[0]

From cf799b1c3630be813d80d7ad49a57b96593f4c07 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 12 Apr 2026 18:57:43 +0200
Subject: [PATCH 056/157] Revert partially deterministic params for Qwen2.5Omni

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 5fd61a6..637e8b1 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -70,6 +70,7 @@ def __init__(self, config: SimpleNamespace):
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
         self.use_video =  getattr(self.config, "use_video", False)
         self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.05)
+        self.temperature = getattr(self.config, "temperature", 1.0)
         self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 5)
 
     @classmethod
@@ -172,8 +173,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
-            do_sample=False,
-            temperature=0.0,
+            temperature=self.temperature,
         )
         if isinstance(output, tuple):
             output = output[0]

From 414bf32416527f5d2e7825fd4a84b31ba987cc07 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 12 Apr 2026 19:13:12 +0200
Subject: [PATCH 057/157] Revert partially deterministic params for
 Phi4Multimodal

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index cffb0e9..8af24c4 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -117,7 +117,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output_attentions=True,
             return_dict_in_generate=True,
             do_sample=False,
-            temperature=0.0,
         )
 
         # Decode newly generated tokens only ──────────────────────────────────────────────────────

From 012b5ca6ecbc203663608474614fff4dfd82b77e Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 13 Apr 2026 11:36:31 +0200
Subject: [PATCH 058/157] Disable strip

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 2 +-
 simulstream/server/speech_processors/qwen2_5_doa.py        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 8af24c4..e7343de 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -157,4 +157,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens).strip()
+        return "".join(tokens)
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 637e8b1..114e227 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -206,4 +206,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         return new_tokens, cross_attn
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens).strip()
+        return "".join(tokens)

From 743233b55e58ae5ce38efd2964a8ab84319c7e99 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 13 Apr 2026 11:43:37 +0200
Subject: [PATCH 059/157] Correct prefix handling

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 114e227..72db619 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -113,16 +113,23 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             },
         ]
 
+        prefix = "".join(self.text_history) if self.text_history else ""
+        if prefix:
+            conversation.append({
+                "role": "assistant",
+                "content": [{"type": "text", "text": prefix}],
+            })
+
         prompt = self.processor.apply_chat_template(
             conversation,
-            add_generation_prompt=True,
+            add_generation_prompt=not bool(prefix),  # False when prefix is present
             tokenize=False,
         )
-        prefix = "".join(self.text_history) if self.text_history else ""
+
         audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
 
         return self.processor(
-            text=f"{prompt}{prefix}",
+            text=prompt,
             audio=audios,
             images=images,
             videos=videos,

From d1a32d55f551bf0b97dff2e2664ef9105d204654 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 14 Apr 2026 18:08:36 +0200
Subject: [PATCH 060/157] Add summary policy

---
 ...phi4multimodal_doa_summary_fixedwords.yaml |  15 ++
 ...i4multimodal_doa_summary_punctuation.yaml} |   9 +-
 .../server/speech_processors/base_doa.py      | 128 +++++++++++++++++-
 .../speech_processors/phi4multimodal_doa.py   |  17 ++-
 .../server/speech_processors/qwen2_5_doa.py   |  34 ++++-
 5 files changed, 195 insertions(+), 8 deletions(-)
 create mode 100644 config/phi4multimodal_doa_summary_fixedwords.yaml
 rename config/{phi4multimodal_doa_frame4.yaml => phi4multimodal_doa_summary_punctuation.yaml} (67%)

diff --git a/config/phi4multimodal_doa_summary_fixedwords.yaml b/config/phi4multimodal_doa_summary_fixedwords.yaml
new file mode 100644
index 0000000..e633004
--- /dev/null
+++ b/config/phi4multimodal_doa_summary_fixedwords.yaml
@@ -0,0 +1,15 @@
+type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_doa.SummaryFixedWordsTextHistory"
+  history_words: __HISTORY__
+  summary_max_new_tokens: 64
+audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+cutoff_frame_num: __FRAME__
+average_attn_over_layers: True
+detokenizer_type: "hf"
+hf_model_name: "microsoft/Phi-4-multimodal-instruct"
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file
diff --git a/config/phi4multimodal_doa_frame4.yaml b/config/phi4multimodal_doa_summary_punctuation.yaml
similarity index 67%
rename from config/phi4multimodal_doa_frame4.yaml
rename to config/phi4multimodal_doa_summary_punctuation.yaml
index ee51a96..542f45a 100644
--- a/config/phi4multimodal_doa_frame4.yaml
+++ b/config/phi4multimodal_doa_summary_punctuation.yaml
@@ -1,12 +1,13 @@
 type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
 text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
-  history_words: 10
+  type: "simulstream.server.speech_processors.base_doa.SummaryPunctuationTextHistory"
+  summary_max_new_tokens: 64
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
-attn_layer: 3
-cutoff_frame_num: 4
+attn_layer: __LAYER__
+cutoff_frame_num: __FRAME__
+average_attn_over_layers: True
 detokenizer_type: "hf"
 hf_model_name: "microsoft/Phi-4-multimodal-instruct"
 word_level_postprocess: True  # Disable if character-level language
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 0f80055..ff75f4c 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -21,7 +21,11 @@
 import torch
 
 from simulstream.server.speech_processors import SAMPLE_RATE
-from simulstream.server.speech_processors.base_streamatt import BaseStreamAtt
+from simulstream.server.speech_processors.base_streamatt import (
+    BaseStreamAtt,
+    FixedWordsTextHistory,
+    PunctuationTextHistory,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -64,6 +68,10 @@ class DecoderOnlyAttention(BaseStreamAtt):
             Default: ``180`` (seconds).
         max_new_tokens : int
             Maximum tokens to generate per chunk.  Default: ``32``.
+        text_history.summary_max_new_tokens : int
+            Extra option used only by summary-capable text-history classes.
+            Maximum tokens generated when updating the running summary.
+            Default: ``64``.
     """
 
     def __init__(self, config: SimpleNamespace):
@@ -72,7 +80,8 @@ def __init__(self, config: SimpleNamespace):
         self.average_attn_over_layers = getattr(self.config, "average_attn_over_layers", False)
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
+        self.max_new_tokens = getattr(self.config, "max_new_tokens", 64)
+        self.prefix_summary = ""
 
     @property
     def audio_max_len(self) -> int:
@@ -127,12 +136,46 @@ def tokens_to_string(self, tokens: List[str]) -> str:
         """Convert a list of decoded tokens to a plain output string."""
         ...
 
+    def summarize_text(self, prompt: str, max_new_tokens: int) -> str:
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement summarize_text()."
+        )
+
     def set_target_language(self, language: str) -> None:
         self.tgt_lang = language
 
     def set_source_language(self, language: str) -> None:
         self.src_lang = language
 
+    def _summary_language_name(self) -> str:
+        if getattr(self, "tgt_lang", None):
+            return LANG_MAPPER.get(self.tgt_lang, self.tgt_lang)
+        if getattr(self, "src_lang", None):
+            return LANG_MAPPER.get(self.src_lang, self.src_lang)
+        return "the same language as the context"
+
+    def build_text_prefix(self) -> str:
+        raw_prefix = "".join(self.text_history) if self.text_history else ""
+        build_text_prefix = getattr(self.text_history_method, "build_text_prefix", None)
+        if build_text_prefix is None:
+            return raw_prefix
+        return build_text_prefix(raw_prefix, self.prefix_summary)
+
+    def _update_text_history(self, new_output: List[str]) -> int:
+        previous_history = list(self.text_history) if self.text_history else []
+        current_history = previous_history + new_output
+        discarded_text = super()._update_text_history(new_output)
+        update_prefix_summary = getattr(self.text_history_method, "update_prefix_summary", None)
+        if discarded_text > 0 and update_prefix_summary is not None:
+            self.prefix_summary = update_prefix_summary(
+                discarded_tokens=current_history[:discarded_text],
+                prefix_summary=self.prefix_summary,
+                tokens_to_string=self.tokens_to_string,
+                summarize_text=self.summarize_text,
+                language_name=self._summary_language_name(),
+            )
+        return discarded_text
+
     def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
         if self.average_attn_over_layers:
             return torch.stack(
@@ -156,3 +199,84 @@ def _preprocess(self, waveform: np.float32) -> dict:
             self.audio_history = self.audio_history[-self.audio_max_len:]
 
         return self.build_processor_inputs(self.audio_history)
+
+    def clear(self) -> None:
+        super().clear()
+        self.prefix_summary = ""
+
+
+class _SummaryPrefixTextHistory:
+    STRONG_PUNCTUATION = [".", "!", "?", ":", ";", "。"]
+
+    def __init__(self, config: SimpleNamespace, _bow_prefix: str):
+        self.summary_max_new_tokens = getattr(config, "summary_max_new_tokens", 32)
+
+    def build_text_prefix(self, raw_prefix: str, prefix_summary: str) -> str:
+        if not prefix_summary:
+            return raw_prefix
+        prefix_summary = prefix_summary.strip()
+        if prefix_summary and not prefix_summary.endswith(tuple(self.STRONG_PUNCTUATION)):
+            prefix_summary = f"{prefix_summary}."
+        return f"{prefix_summary} {raw_prefix}" if raw_prefix else f"{prefix_summary} "
+
+    def update_prefix_summary(
+            self,
+            discarded_tokens: List[str],
+            prefix_summary: str,
+            tokens_to_string,
+            summarize_text,
+            language_name: str) -> str:
+        discarded_text = tokens_to_string(discarded_tokens).strip()
+        if not discarded_text:
+            return prefix_summary
+        if prefix_summary:
+            summary_prompt = (
+                f"Update this running summary in {language_name}. Keep it brief and useful for "
+                f"continuing the translation. Preserve key entities, terminology, numbers, and "
+                f"unresolved references. Return only the updated summary.\n\n"
+                f"Current summary:\n{prefix_summary}\n\n"
+                f"New earlier context:\n{discarded_text}"
+            )
+        else:
+            summary_prompt = (
+                f"Summarize the following earlier context in {language_name}. Keep it brief and "
+                f"useful for continuing the translation. Preserve key entities, terminology, "
+                f"numbers, and unresolved references. Return only the summary.\n\n"
+                f"Earlier context:\n{discarded_text}"
+            )
+        new_summary = summarize_text(
+            summary_prompt,
+            self.summary_max_new_tokens,
+        ).strip()
+        return new_summary or prefix_summary
+
+
+class SummaryFixedWordsTextHistory(FixedWordsTextHistory, _SummaryPrefixTextHistory):
+    """
+    Fixed-words text-history selector plus a DOA-only running summary prefix.
+
+    Config attributes
+    -----------------
+    history_words : int
+        Number of recent raw words to retain for StreamAtt alignment.
+    summary_max_new_tokens : int
+        Maximum tokens used when updating the running summary.
+    """
+
+    def __init__(self, config: SimpleNamespace, bow_prefix: str):
+        FixedWordsTextHistory.__init__(self, config, bow_prefix)
+        _SummaryPrefixTextHistory.__init__(self, config, bow_prefix)
+
+
+class SummaryPunctuationTextHistory(PunctuationTextHistory, _SummaryPrefixTextHistory):
+    """
+    Punctuation-based text-history selector plus a DOA-only running summary prefix.
+
+    The raw retained history still follows the punctuation selector, while the
+    discarded older context is compressed into a running summary for the next
+    decoder-only prompt.
+    """
+
+    def __init__(self, config: SimpleNamespace, bow_prefix: str):
+        PunctuationTextHistory.__init__(self, config, bow_prefix)
+        _SummaryPrefixTextHistory.__init__(self, config, bow_prefix)
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index e7343de..516dceb 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -67,7 +67,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
 
     def build_prompt(self) -> str:
         filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
-        prefix = "".join(self.text_history) if self.text_history else ""
+        prefix = self.build_text_prefix()
         prompt = (
             f"{self._USER_START}{self._AUDIO_TOKEN}"
             f"{filled_prompt}{self._END_TOKEN}"
@@ -82,6 +82,21 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             return_tensors="pt",
         ).to(self.device)
 
+    def summarize_text(self, prompt: str, max_new_tokens: int) -> str:
+        inputs = self.processor(
+            text=f"{self._USER_START}{prompt}{self._END_TOKEN}{self._ASST_START}",
+            return_tensors="pt",
+        ).to(self.device)
+        output = self.model.generate(
+            **inputs,
+            max_new_tokens=max_new_tokens,
+            generation_config=self.generation_config,
+            num_logits_to_keep=1,
+            do_sample=False,
+        )
+        new_ids = output[:, inputs["input_ids"].shape[1]:]
+        return self.processor.tokenizer.decode(new_ids[0], skip_special_tokens=True).strip()
+
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         """
         Run greedy generation and build the proxy cross-attention matrix.
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 72db619..28365f1 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -113,7 +113,7 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             },
         ]
 
-        prefix = "".join(self.text_history) if self.text_history else ""
+        prefix = self.build_text_prefix()
         if prefix:
             conversation.append({
                 "role": "assistant",
@@ -139,6 +139,38 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             use_audio_in_video=True,
         ).to(self.device)
 
+    def summarize_text(self, prompt: str, max_new_tokens: int) -> str:
+        conversation = [
+            {
+                "role": "system",
+                "content": [{"type": "text", "text": self.SYSTEM_PROMPT}],
+            },
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": prompt}],
+            },
+        ]
+        summary_prompt = self.processor.apply_chat_template(
+            conversation,
+            add_generation_prompt=True,
+            tokenize=False,
+        )
+        inputs = self.processor(
+            text=summary_prompt,
+            return_tensors="pt",
+            padding=True,
+        ).to(self.device)
+        output = self.model.generate(
+            **inputs,
+            return_audio=False,
+            thinker_max_new_tokens=max_new_tokens,
+            thinker_do_sample=False,
+        )
+        if isinstance(output, tuple):
+            output = output[0]
+        new_ids = output[:, inputs["input_ids"].shape[1]:]
+        return self.processor.tokenizer.decode(new_ids[0], skip_special_tokens=True).strip()
+
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
         if audio_positions.numel() > 0:

From 6fdde365e201f8865e39a0e85294244464eda002 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 14 Apr 2026 18:15:56 +0200
Subject: [PATCH 061/157] Revert to original implementation of the prefix for
 Qwen

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 28365f1..aab2dbf 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -113,23 +113,17 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             },
         ]
 
-        prefix = self.build_text_prefix()
-        if prefix:
-            conversation.append({
-                "role": "assistant",
-                "content": [{"type": "text", "text": prefix}],
-            })
-
         prompt = self.processor.apply_chat_template(
             conversation,
-            add_generation_prompt=not bool(prefix),  # False when prefix is present
+            add_generation_prompt=True,
             tokenize=False,
         )
+        prefix = self.build_text_prefix()
 
         audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
 
         return self.processor(
-            text=prompt,
+            text=f"{prompt}{prefix}",
             audio=audios,
             images=images,
             videos=videos,

From 69ddeef844846d0dece9c49f4560cadef02fe2ad Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 15 Apr 2026 17:03:16 +0200
Subject: [PATCH 062/157] Try the fix

---
 .../server/speech_processors/qwen2_5_doa.py   | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index aab2dbf..0d324f8 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -68,6 +68,7 @@ def __init__(self, config: SimpleNamespace):
         text_history_cls = class_load(self.text_history_config.type)
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
+        self.prefix_token_count = 0
         self.use_video =  getattr(self.config, "use_video", False)
         self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.05)
         self.temperature = getattr(self.config, "temperature", 1.0)
@@ -119,11 +120,24 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             tokenize=False,
         )
         prefix = self.build_text_prefix()
+        full_prompt = f"{prompt}{prefix}"
+        if prefix:
+            prompt_ids = self.processor.tokenizer(
+                prompt,
+                add_special_tokens=False,
+            )["input_ids"]
+            full_prompt_ids = self.processor.tokenizer(
+                full_prompt,
+                add_special_tokens=False,
+            )["input_ids"]
+            self.prefix_token_count = max(0, len(full_prompt_ids) - len(prompt_ids))
+        else:
+            self.prefix_token_count = 0
 
         audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
 
         return self.processor(
-            text=f"{prompt}{prefix}",
+            text=full_prompt,
             audio=audios,
             images=images,
             videos=videos,
@@ -218,7 +232,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         ]
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        prefix_len = len(self.text_history) if self.text_history else 0
+        prefix_len = self.prefix_token_count
         if prefix_len > 0:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
         else:
@@ -240,3 +254,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
     def tokens_to_string(self, tokens: List[str]) -> str:
         return "".join(tokens)
+
+    def clear(self) -> None:
+        super().clear()
+        self.prefix_token_count = 0

From f964e47fb33f7fbd68f56a0a6ed71e483b8c08bd Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 15 Apr 2026 17:25:29 +0200
Subject: [PATCH 063/157] Revert "Try the fix"

This reverts commit 69ddeef844846d0dece9c49f4560cadef02fe2ad.
---
 .../server/speech_processors/qwen2_5_doa.py   | 22 ++-----------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 0d324f8..aab2dbf 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -68,7 +68,6 @@ def __init__(self, config: SimpleNamespace):
         text_history_cls = class_load(self.text_history_config.type)
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
-        self.prefix_token_count = 0
         self.use_video =  getattr(self.config, "use_video", False)
         self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.05)
         self.temperature = getattr(self.config, "temperature", 1.0)
@@ -120,24 +119,11 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             tokenize=False,
         )
         prefix = self.build_text_prefix()
-        full_prompt = f"{prompt}{prefix}"
-        if prefix:
-            prompt_ids = self.processor.tokenizer(
-                prompt,
-                add_special_tokens=False,
-            )["input_ids"]
-            full_prompt_ids = self.processor.tokenizer(
-                full_prompt,
-                add_special_tokens=False,
-            )["input_ids"]
-            self.prefix_token_count = max(0, len(full_prompt_ids) - len(prompt_ids))
-        else:
-            self.prefix_token_count = 0
 
         audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
 
         return self.processor(
-            text=full_prompt,
+            text=f"{prompt}{prefix}",
             audio=audios,
             images=images,
             videos=videos,
@@ -232,7 +218,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         ]
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        prefix_len = self.prefix_token_count
+        prefix_len = len(self.text_history) if self.text_history else 0
         if prefix_len > 0:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
         else:
@@ -254,7 +240,3 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
     def tokens_to_string(self, tokens: List[str]) -> str:
         return "".join(tokens)
-
-    def clear(self) -> None:
-        super().clear()
-        self.prefix_token_count = 0

From f117590fba0667c7f0bd1b2e88a633c9f0056ce5 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 16 Apr 2026 10:18:05 +0200
Subject: [PATCH 064/157] Add Qwen2.5omni 7b config

---
 config/qwen2.5omni_7b_doa_punctuation.yaml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 config/qwen2.5omni_7b_doa_punctuation.yaml

diff --git a/config/qwen2.5omni_7b_doa_punctuation.yaml b/config/qwen2.5omni_7b_doa_punctuation.yaml
new file mode 100644
index 0000000..bdb0e06
--- /dev/null
+++ b/config/qwen2.5omni_7b_doa_punctuation.yaml
@@ -0,0 +1,14 @@
+type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
+detokenizer_type: "hf"
+hf_model_name: "Qwen/Qwen2.5-Omni-7B"
+use_video: False
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file

From b9830a99232dd3d387e5da1b8c39b75ab8259703 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 16 Apr 2026 18:33:37 +0200
Subject: [PATCH 065/157] reduce audio max len

---
 config/qwen2.5omni_7b_doa_punctuation.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/qwen2.5omni_7b_doa_punctuation.yaml b/config/qwen2.5omni_7b_doa_punctuation.yaml
index bdb0e06..ed5cc42 100644
--- a/config/qwen2.5omni_7b_doa_punctuation.yaml
+++ b/config/qwen2.5omni_7b_doa_punctuation.yaml
@@ -1,7 +1,7 @@
 type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
+audio_history_max_duration: 90  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__

From fbb551753799312cecf899ab32d0ebb89b3d385f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 16 Apr 2026 18:38:19 +0200
Subject: [PATCH 066/157] Move summary

---
 .../speech_processors/phi4multimodal_doa.py   | 27 ++++++++++++++-----
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 516dceb..e17e592 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -67,12 +67,25 @@ def load_model(cls, config: SimpleNamespace) -> None:
 
     def build_prompt(self) -> str:
         filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
-        prefix = self.build_text_prefix()
-        prompt = (
-            f"{self._USER_START}{self._AUDIO_TOKEN}"
-            f"{filled_prompt}{self._END_TOKEN}"
-            f"{self._ASST_START}{prefix}"
-        )
+        summary = self.prefix_summary.strip() if self.prefix_summary else ""
+        raw_prefix = "".join(self.text_history) if self.text_history else ""
+
+        if summary:
+            # Place the summary as context in the user turn so the model does not
+            # interpret it as tokens it has already generated, which would cause it
+            # to emit EOS immediately and produce no new output.
+            prompt = (
+                f"{self._USER_START}{self._AUDIO_TOKEN}"
+                f"Context of what was translated so far: {summary}\n\n"
+                f"{filled_prompt}{self._END_TOKEN}"
+                f"{self._ASST_START}{raw_prefix}"
+            )
+        else:
+            prompt = (
+                f"{self._USER_START}{self._AUDIO_TOKEN}"
+                f"{filled_prompt}{self._END_TOKEN}"
+                f"{self._ASST_START}{raw_prefix}"
+            )
         return prompt
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
@@ -172,4 +185,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
+        return "".join(tokens)
\ No newline at end of file

From e44980abc8b247213bf5c07bcc91aff0ed18e6df Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 5 May 2026 14:56:57 +0200
Subject: [PATCH 067/157] Try change for summary

---
 .../server/speech_processors/base_doa.py      | 28 ++++++----------
 .../speech_processors/phi4multimodal_doa.py   | 32 ++++++++-----------
 .../server/speech_processors/qwen2_5_doa.py   | 15 +++++++--
 3 files changed, 37 insertions(+), 38 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index ff75f4c..4f9a877 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -80,7 +80,7 @@ def __init__(self, config: SimpleNamespace):
         self.average_attn_over_layers = getattr(self.config, "average_attn_over_layers", False)
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        self.max_new_tokens = getattr(self.config, "max_new_tokens", 64)
+        self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
         self.prefix_summary = ""
 
     @property
@@ -154,12 +154,14 @@ def _summary_language_name(self) -> str:
             return LANG_MAPPER.get(self.src_lang, self.src_lang)
         return "the same language as the context"
 
-    def build_text_prefix(self) -> str:
-        raw_prefix = "".join(self.text_history) if self.text_history else ""
-        build_text_prefix = getattr(self.text_history_method, "build_text_prefix", None)
-        if build_text_prefix is None:
-            return raw_prefix
-        return build_text_prefix(raw_prefix, self.prefix_summary)
+    def build_raw_text_prefix(self) -> str:
+        return "".join(self.text_history) if self.text_history else ""
+
+    def build_summary_context(self) -> str:
+        update_prefix_summary = getattr(self.text_history_method, "update_prefix_summary", None)
+        if update_prefix_summary is None:
+            return ""
+        return self.prefix_summary.strip()
 
     def _update_text_history(self, new_output: List[str]) -> int:
         previous_history = list(self.text_history) if self.text_history else []
@@ -206,18 +208,8 @@ def clear(self) -> None:
 
 
 class _SummaryPrefixTextHistory:
-    STRONG_PUNCTUATION = [".", "!", "?", ":", ";", "。"]
-
     def __init__(self, config: SimpleNamespace, _bow_prefix: str):
-        self.summary_max_new_tokens = getattr(config, "summary_max_new_tokens", 32)
-
-    def build_text_prefix(self, raw_prefix: str, prefix_summary: str) -> str:
-        if not prefix_summary:
-            return raw_prefix
-        prefix_summary = prefix_summary.strip()
-        if prefix_summary and not prefix_summary.endswith(tuple(self.STRONG_PUNCTUATION)):
-            prefix_summary = f"{prefix_summary}."
-        return f"{prefix_summary} {raw_prefix}" if raw_prefix else f"{prefix_summary} "
+        self.summary_max_new_tokens = getattr(config, "summary_max_new_tokens", 64)
 
     def update_prefix_summary(
             self,
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index e17e592..6be6b53 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -67,25 +67,21 @@ def load_model(cls, config: SimpleNamespace) -> None:
 
     def build_prompt(self) -> str:
         filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
-        summary = self.prefix_summary.strip() if self.prefix_summary else ""
-        raw_prefix = "".join(self.text_history) if self.text_history else ""
-
+        summary = self.build_summary_context()
+        raw_prefix = self.build_raw_text_prefix()
         if summary:
-            # Place the summary as context in the user turn so the model does not
-            # interpret it as tokens it has already generated, which would cause it
-            # to emit EOS immediately and produce no new output.
-            prompt = (
-                f"{self._USER_START}{self._AUDIO_TOKEN}"
-                f"Context of what was translated so far: {summary}\n\n"
-                f"{filled_prompt}{self._END_TOKEN}"
-                f"{self._ASST_START}{raw_prefix}"
-            )
-        else:
-            prompt = (
-                f"{self._USER_START}{self._AUDIO_TOKEN}"
-                f"{filled_prompt}{self._END_TOKEN}"
-                f"{self._ASST_START}{raw_prefix}"
+            filled_prompt = (
+                f"Previous translated context summary in {LANG_MAPPER[self.tgt_lang]}: "
+                f"{summary}\n"
+                f"Use this only as context for continuation. Do not repeat or paraphrase the "
+                f"summary.\n\n"
+                f"{filled_prompt}"
             )
+        prompt = (
+            f"{self._USER_START}{self._AUDIO_TOKEN}"
+            f"{filled_prompt}{self._END_TOKEN}"
+            f"{self._ASST_START}{raw_prefix}"
+        )
         return prompt
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
@@ -185,4 +181,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
\ No newline at end of file
+        return "".join(tokens)
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index aab2dbf..1456692 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -99,6 +99,17 @@ def build_prompt(self) -> str:
         )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        summary = self.build_summary_context()
+        prompt_text = self.build_prompt()
+        if summary:
+            prompt_text = (
+                f"Previous translated context summary in {LANG_MAPPER[self.tgt_lang]}: "
+                f"{summary}\n"
+                f"Use this only as context for continuation. Do not repeat or paraphrase the "
+                f"summary.\n\n"
+                f"{prompt_text}"
+            )
+
         conversation = [
             {
                 "role": "system",
@@ -108,7 +119,7 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
                 "role": "user",
                 "content": [
                     {"type": "audio", "audio": waveform},
-                    {"type": "text", "text": self.build_prompt()},
+                    {"type": "text", "text": prompt_text},
                 ],
             },
         ]
@@ -118,7 +129,7 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             add_generation_prompt=True,
             tokenize=False,
         )
-        prefix = self.build_text_prefix()
+        prefix = self.build_raw_text_prefix()
 
         audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
 

From dfcd53e1eeb1aa507bfc0506026af1cf2f0f76a7 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 5 May 2026 18:48:12 +0200
Subject: [PATCH 068/157] Try change for summary

---
 .../server/speech_processors/base_doa.py      | 67 ++++++++++++-------
 .../speech_processors/phi4multimodal_doa.py   |  6 +-
 .../server/speech_processors/qwen2_5_doa.py   |  6 +-
 3 files changed, 49 insertions(+), 30 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 4f9a877..cdf9eee 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -72,6 +72,10 @@ class DecoderOnlyAttention(BaseStreamAtt):
             Extra option used only by summary-capable text-history classes.
             Maximum tokens generated when updating the running summary.
             Default: ``64``.
+        text_history.summary_source_max_tokens : int
+            Maximum number of discarded raw text tokens to keep as source for
+            regenerating the summary from scratch.
+            Default: ``256``.
     """
 
     def __init__(self, config: SimpleNamespace):
@@ -82,6 +86,7 @@ def __init__(self, config: SimpleNamespace):
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
         self.prefix_summary = ""
+        self.summary_source_tokens = []
 
     @property
     def audio_max_len(self) -> int:
@@ -169,9 +174,16 @@ def _update_text_history(self, new_output: List[str]) -> int:
         discarded_text = super()._update_text_history(new_output)
         update_prefix_summary = getattr(self.text_history_method, "update_prefix_summary", None)
         if discarded_text > 0 and update_prefix_summary is not None:
+            self.summary_source_tokens.extend(current_history[:discarded_text])
+            trim_summary_source_tokens = getattr(
+                self.text_history_method,
+                "trim_summary_source_tokens",
+                None,
+            )
+            if trim_summary_source_tokens is not None:
+                self.summary_source_tokens = trim_summary_source_tokens(self.summary_source_tokens)
             self.prefix_summary = update_prefix_summary(
-                discarded_tokens=current_history[:discarded_text],
-                prefix_summary=self.prefix_summary,
+                summary_tokens=self.summary_source_tokens,
                 tokens_to_string=self.tokens_to_string,
                 summarize_text=self.summarize_text,
                 language_name=self._summary_language_name(),
@@ -205,42 +217,51 @@ def _preprocess(self, waveform: np.float32) -> dict:
     def clear(self) -> None:
         super().clear()
         self.prefix_summary = ""
+        self.summary_source_tokens = []
 
 
 class _SummaryPrefixTextHistory:
+    """
+    Base Summary text-history selector.
+
+    Config attributes
+    -----------------
+    summary_max_new_tokens : int
+        Maximum tokens used when updating the running summary.
+    summary_source_max_tokens : int
+        Maximum number of discarded raw text tokens kept as summary source.
+    """
+
     def __init__(self, config: SimpleNamespace, _bow_prefix: str):
         self.summary_max_new_tokens = getattr(config, "summary_max_new_tokens", 64)
+        self.summary_source_max_tokens = getattr(config, "summary_source_max_tokens", 256)
+
+    def trim_summary_source_tokens(self, summary_tokens: List[str]) -> List[str]:
+        if len(summary_tokens) <= self.summary_source_max_tokens:
+            return summary_tokens
+        return summary_tokens[-self.summary_source_max_tokens:]
 
     def update_prefix_summary(
             self,
-            discarded_tokens: List[str],
-            prefix_summary: str,
+            summary_tokens: List[str],
             tokens_to_string,
             summarize_text,
             language_name: str) -> str:
-        discarded_text = tokens_to_string(discarded_tokens).strip()
+        discarded_text = tokens_to_string(summary_tokens).strip()
         if not discarded_text:
-            return prefix_summary
-        if prefix_summary:
-            summary_prompt = (
-                f"Update this running summary in {language_name}. Keep it brief and useful for "
-                f"continuing the translation. Preserve key entities, terminology, numbers, and "
-                f"unresolved references. Return only the updated summary.\n\n"
-                f"Current summary:\n{prefix_summary}\n\n"
-                f"New earlier context:\n{discarded_text}"
-            )
-        else:
-            summary_prompt = (
-                f"Summarize the following earlier context in {language_name}. Keep it brief and "
-                f"useful for continuing the translation. Preserve key entities, terminology, "
-                f"numbers, and unresolved references. Return only the summary.\n\n"
-                f"Earlier context:\n{discarded_text}"
-            )
+            return ""
+        summary_prompt = (
+            f"Summarize the following earlier translated context in {language_name}. "
+            f"Return short memory notes useful for continuing the translation. Preserve key "
+            f"entities, terminology, abbreviations, numbers, and unresolved references. "
+            f"Prefer concise notes over full prose. Return only the memory notes.\n\n"
+            f"Earlier translated context:\n{discarded_text}"
+        )
         new_summary = summarize_text(
             summary_prompt,
             self.summary_max_new_tokens,
         ).strip()
-        return new_summary or prefix_summary
+        return new_summary
 
 
 class SummaryFixedWordsTextHistory(FixedWordsTextHistory, _SummaryPrefixTextHistory):
@@ -251,8 +272,6 @@ class SummaryFixedWordsTextHistory(FixedWordsTextHistory, _SummaryPrefixTextHist
     -----------------
     history_words : int
         Number of recent raw words to retain for StreamAtt alignment.
-    summary_max_new_tokens : int
-        Maximum tokens used when updating the running summary.
     """
 
     def __init__(self, config: SimpleNamespace, bow_prefix: str):
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 6be6b53..bbc2ad0 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -71,10 +71,10 @@ def build_prompt(self) -> str:
         raw_prefix = self.build_raw_text_prefix()
         if summary:
             filled_prompt = (
-                f"Previous translated context summary in {LANG_MAPPER[self.tgt_lang]}: "
+                f"Background memory from earlier translated audio in {LANG_MAPPER[self.tgt_lang]}: "
                 f"{summary}\n"
-                f"Use this only as context for continuation. Do not repeat or paraphrase the "
-                f"summary.\n\n"
+                f"Continue the existing translation in {LANG_MAPPER[self.tgt_lang]}. "
+                f"Output only the next continuation.\n\n"
                 f"{filled_prompt}"
             )
         prompt = (
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 1456692..426c7f3 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -103,10 +103,10 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         prompt_text = self.build_prompt()
         if summary:
             prompt_text = (
-                f"Previous translated context summary in {LANG_MAPPER[self.tgt_lang]}: "
+                f"Background memory from earlier translated audio in {LANG_MAPPER[self.tgt_lang]}: "
                 f"{summary}\n"
-                f"Use this only as context for continuation. Do not repeat or paraphrase the "
-                f"summary.\n\n"
+                f"Continue the existing translation in {LANG_MAPPER[self.tgt_lang]}. "
+                f"Output only the next continuation.\n\n"
                 f"{prompt_text}"
             )
 

From 3989154b04ea5c434ef43b952290b9f7f6743f06 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 5 May 2026 21:34:52 +0200
Subject: [PATCH 069/157] Revert

---
 ...phi4multimodal_doa_summary_fixedwords.yaml |  4 +-
 ...hi4multimodal_doa_summary_punctuation.yaml |  4 +-
 .../server/speech_processors/base_doa.py      | 65 ++++++++-----------
 3 files changed, 30 insertions(+), 43 deletions(-)

diff --git a/config/phi4multimodal_doa_summary_fixedwords.yaml b/config/phi4multimodal_doa_summary_fixedwords.yaml
index e633004..fdccaff 100644
--- a/config/phi4multimodal_doa_summary_fixedwords.yaml
+++ b/config/phi4multimodal_doa_summary_fixedwords.yaml
@@ -2,7 +2,7 @@ type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA
 text_history:
   type: "simulstream.server.speech_processors.base_doa.SummaryFixedWordsTextHistory"
   history_words: __HISTORY__
-  summary_max_new_tokens: 64
+  summary_max_new_tokens: 32
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
@@ -12,4 +12,4 @@ average_attn_over_layers: True
 detokenizer_type: "hf"
 hf_model_name: "microsoft/Phi-4-multimodal-instruct"
 word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
+max_new_tokens: 32
diff --git a/config/phi4multimodal_doa_summary_punctuation.yaml b/config/phi4multimodal_doa_summary_punctuation.yaml
index 542f45a..15d2e9b 100644
--- a/config/phi4multimodal_doa_summary_punctuation.yaml
+++ b/config/phi4multimodal_doa_summary_punctuation.yaml
@@ -1,7 +1,7 @@
 type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_doa.SummaryPunctuationTextHistory"
-  summary_max_new_tokens: 64
+  summary_max_new_tokens: 32
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
@@ -11,4 +11,4 @@ average_attn_over_layers: True
 detokenizer_type: "hf"
 hf_model_name: "microsoft/Phi-4-multimodal-instruct"
 word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
+max_new_tokens: 32
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index cdf9eee..f316819 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -68,14 +68,6 @@ class DecoderOnlyAttention(BaseStreamAtt):
             Default: ``180`` (seconds).
         max_new_tokens : int
             Maximum tokens to generate per chunk.  Default: ``32``.
-        text_history.summary_max_new_tokens : int
-            Extra option used only by summary-capable text-history classes.
-            Maximum tokens generated when updating the running summary.
-            Default: ``64``.
-        text_history.summary_source_max_tokens : int
-            Maximum number of discarded raw text tokens to keep as source for
-            regenerating the summary from scratch.
-            Default: ``256``.
     """
 
     def __init__(self, config: SimpleNamespace):
@@ -86,7 +78,6 @@ def __init__(self, config: SimpleNamespace):
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
         self.prefix_summary = ""
-        self.summary_source_tokens = []
 
     @property
     def audio_max_len(self) -> int:
@@ -174,20 +165,15 @@ def _update_text_history(self, new_output: List[str]) -> int:
         discarded_text = super()._update_text_history(new_output)
         update_prefix_summary = getattr(self.text_history_method, "update_prefix_summary", None)
         if discarded_text > 0 and update_prefix_summary is not None:
-            self.summary_source_tokens.extend(current_history[:discarded_text])
-            trim_summary_source_tokens = getattr(
-                self.text_history_method,
-                "trim_summary_source_tokens",
-                None,
-            )
-            if trim_summary_source_tokens is not None:
-                self.summary_source_tokens = trim_summary_source_tokens(self.summary_source_tokens)
-            self.prefix_summary = update_prefix_summary(
-                summary_tokens=self.summary_source_tokens,
+            new_summary = update_prefix_summary(
+                prefix_summary=self.prefix_summary,
+                discarded_tokens=current_history[:discarded_text],
                 tokens_to_string=self.tokens_to_string,
                 summarize_text=self.summarize_text,
                 language_name=self._summary_language_name(),
             )
+            if new_summary:
+                self.prefix_summary = new_summary
         return discarded_text
 
     def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
@@ -217,46 +203,47 @@ def _preprocess(self, waveform: np.float32) -> dict:
     def clear(self) -> None:
         super().clear()
         self.prefix_summary = ""
-        self.summary_source_tokens = []
 
 
 class _SummaryPrefixTextHistory:
     """
-    Base Summary text-history selector.
+    Base summary text-history selector.
 
     Config attributes
     -----------------
     summary_max_new_tokens : int
         Maximum tokens used when updating the running summary.
-    summary_source_max_tokens : int
-        Maximum number of discarded raw text tokens kept as summary source.
     """
 
     def __init__(self, config: SimpleNamespace, _bow_prefix: str):
-        self.summary_max_new_tokens = getattr(config, "summary_max_new_tokens", 64)
-        self.summary_source_max_tokens = getattr(config, "summary_source_max_tokens", 256)
-
-    def trim_summary_source_tokens(self, summary_tokens: List[str]) -> List[str]:
-        if len(summary_tokens) <= self.summary_source_max_tokens:
-            return summary_tokens
-        return summary_tokens[-self.summary_source_max_tokens:]
+        self.summary_max_new_tokens = getattr(config, "summary_max_new_tokens", 32)
 
     def update_prefix_summary(
             self,
-            summary_tokens: List[str],
+            prefix_summary: str,
+            discarded_tokens: List[str],
             tokens_to_string,
             summarize_text,
             language_name: str) -> str:
-        discarded_text = tokens_to_string(summary_tokens).strip()
+        discarded_text = tokens_to_string(discarded_tokens).strip()
         if not discarded_text:
             return ""
-        summary_prompt = (
-            f"Summarize the following earlier translated context in {language_name}. "
-            f"Return short memory notes useful for continuing the translation. Preserve key "
-            f"entities, terminology, abbreviations, numbers, and unresolved references. "
-            f"Prefer concise notes over full prose. Return only the memory notes.\n\n"
-            f"Earlier translated context:\n{discarded_text}"
-        )
+        if prefix_summary:
+            summary_prompt = (
+                f"Update these memory notes in {language_name}. Keep them brief and useful for "
+                f"continuing the translation. Preserve key entities, terminology, abbreviations, "
+                f"numbers, and unresolved references. Return only the updated memory notes.\n\n"
+                f"Current memory notes:\n{prefix_summary}\n\n"
+                f"New earlier translated context:\n{discarded_text}"
+            )
+        else:
+            summary_prompt = (
+                f"Summarize the following earlier translated context in {language_name}. "
+                f"Return short memory notes useful for continuing the translation. Preserve key "
+                f"entities, terminology, abbreviations, numbers, and unresolved references. "
+                f"Prefer concise notes over full prose. Return only the memory notes.\n\n"
+                f"Earlier translated context:\n{discarded_text}"
+            )
         new_summary = summarize_text(
             summary_prompt,
             self.summary_max_new_tokens,

From 5a8237197953b9d0579da4a86f16ec7d3117fe02 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 5 May 2026 21:36:22 +0200
Subject: [PATCH 070/157] Try fix

---
 .../speech_processors/phi4multimodal_doa.py   | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index bbc2ad0..c9a7706 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -145,10 +145,16 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         # Decode newly generated tokens only ──────────────────────────────────────────────────────
         new_ids = output.sequences[:, input_len:]  # (1, n_new)
-        new_tokens = [
-            self.processor.tokenizer.decode([t], skip_special_tokens=True)
-            for t in new_ids[0]
-        ]
+        eos_id = self.generation_config.eos_token_id
+        eos_ids = {eos_id} if isinstance(eos_id, int) else set(eos_id or [])
+        truncated_ids = []
+        for t in new_ids[0]:
+            if t.item() in eos_ids:
+                break
+            truncated_ids.append(t)
+        new_tokens = self.processor.tokenizer.convert_ids_to_tokens(
+            truncated_ids, skip_special_tokens=True
+        ) if truncated_ids else []
 
         # Build proxy cross-attention for the hypothesis (prefix + new_tokens) ────────────────────
         # Prefix rows from the prefill pass
@@ -163,12 +169,13 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
         # The prefill pass predicts the first generated token, so we use the last prompt row
         # as its proxy audio-attention. Subsequent generated tokens come from later decode steps.
-        first_new_row = prefill_attn[-1:, audio_positions] if len(new_tokens) > 0 else \
+        n_new = len(truncated_ids)
+        first_new_row = prefill_attn[-1:, audio_positions] if n_new > 0 else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn)
             .squeeze(0)[audio_positions]  # (audio_len,)
-            for step_attn in output.attentions[1:]
+            for step_attn in output.attentions[1:n_new + 1]
         ]
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
@@ -181,4 +188,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
+        return self.processor.tokenizer.convert_tokens_to_string(tokens)
\ No newline at end of file

From abefe053eb22a615220d29f48b3392bb25b33612 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 5 May 2026 21:43:01 +0200
Subject: [PATCH 071/157] Revert "Try fix"

This reverts commit 5a8237197953b9d0579da4a86f16ec7d3117fe02.
---
 .../speech_processors/phi4multimodal_doa.py   | 21 +++++++------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index c9a7706..bbc2ad0 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -145,16 +145,10 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         # Decode newly generated tokens only ──────────────────────────────────────────────────────
         new_ids = output.sequences[:, input_len:]  # (1, n_new)
-        eos_id = self.generation_config.eos_token_id
-        eos_ids = {eos_id} if isinstance(eos_id, int) else set(eos_id or [])
-        truncated_ids = []
-        for t in new_ids[0]:
-            if t.item() in eos_ids:
-                break
-            truncated_ids.append(t)
-        new_tokens = self.processor.tokenizer.convert_ids_to_tokens(
-            truncated_ids, skip_special_tokens=True
-        ) if truncated_ids else []
+        new_tokens = [
+            self.processor.tokenizer.decode([t], skip_special_tokens=True)
+            for t in new_ids[0]
+        ]
 
         # Build proxy cross-attention for the hypothesis (prefix + new_tokens) ────────────────────
         # Prefix rows from the prefill pass
@@ -169,13 +163,12 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
         # The prefill pass predicts the first generated token, so we use the last prompt row
         # as its proxy audio-attention. Subsequent generated tokens come from later decode steps.
-        n_new = len(truncated_ids)
-        first_new_row = prefill_attn[-1:, audio_positions] if n_new > 0 else \
+        first_new_row = prefill_attn[-1:, audio_positions] if len(new_tokens) > 0 else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn)
             .squeeze(0)[audio_positions]  # (audio_len,)
-            for step_attn in output.attentions[1:n_new + 1]
+            for step_attn in output.attentions[1:]
         ]
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
@@ -188,4 +181,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return self.processor.tokenizer.convert_tokens_to_string(tokens)
\ No newline at end of file
+        return "".join(tokens)

From 73eda2ace7cc46cf96ac706cc23f4909c1d596a0 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 6 May 2026 18:24:43 +0200
Subject: [PATCH 072/157] Switch from summary to reference memory

---
 ...4multimodal_doa_reference_fixedwords.yaml} |   3 +-
 ...multimodal_doa_reference_punctuation.yaml} |   3 +-
 .../server/speech_processors/base_doa.py      | 113 +++++++++---------
 .../speech_processors/phi4multimodal_doa.py   |  15 +--
 .../server/speech_processors/qwen2_5_doa.py   |  19 +--
 5 files changed, 78 insertions(+), 75 deletions(-)
 rename config/{phi4multimodal_doa_summary_fixedwords.yaml => phi4multimodal_doa_reference_fixedwords.yaml} (81%)
 rename config/{phi4multimodal_doa_summary_punctuation.yaml => phi4multimodal_doa_reference_punctuation.yaml} (80%)

diff --git a/config/phi4multimodal_doa_summary_fixedwords.yaml b/config/phi4multimodal_doa_reference_fixedwords.yaml
similarity index 81%
rename from config/phi4multimodal_doa_summary_fixedwords.yaml
rename to config/phi4multimodal_doa_reference_fixedwords.yaml
index fdccaff..75b501a 100644
--- a/config/phi4multimodal_doa_summary_fixedwords.yaml
+++ b/config/phi4multimodal_doa_reference_fixedwords.yaml
@@ -1,8 +1,7 @@
 type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
 text_history:
-  type: "simulstream.server.speech_processors.base_doa.SummaryFixedWordsTextHistory"
+  type: "simulstream.server.speech_processors.base_doa.ReferenceMemoryFixedWordsTextHistory"
   history_words: __HISTORY__
-  summary_max_new_tokens: 32
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
diff --git a/config/phi4multimodal_doa_summary_punctuation.yaml b/config/phi4multimodal_doa_reference_punctuation.yaml
similarity index 80%
rename from config/phi4multimodal_doa_summary_punctuation.yaml
rename to config/phi4multimodal_doa_reference_punctuation.yaml
index 15d2e9b..66bea36 100644
--- a/config/phi4multimodal_doa_summary_punctuation.yaml
+++ b/config/phi4multimodal_doa_reference_punctuation.yaml
@@ -1,7 +1,6 @@
 type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
 text_history:
-  type: "simulstream.server.speech_processors.base_doa.SummaryPunctuationTextHistory"
-  summary_max_new_tokens: 32
+  type: "simulstream.server.speech_processors.base_doa.ReferenceMemoryPunctuationTextHistory"
 audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index f316819..a63a52c 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -77,7 +77,7 @@ def __init__(self, config: SimpleNamespace):
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
-        self.prefix_summary = ""
+        self.prefix_memory = ""
 
     @property
     def audio_max_len(self) -> int:
@@ -132,9 +132,9 @@ def tokens_to_string(self, tokens: List[str]) -> str:
         """Convert a list of decoded tokens to a plain output string."""
         ...
 
-    def summarize_text(self, prompt: str, max_new_tokens: int) -> str:
+    def generate_text_completion(self, prompt: str, max_new_tokens: int) -> str:
         raise NotImplementedError(
-            f"{self.__class__.__name__} does not implement summarize_text()."
+            f"{self.__class__.__name__} does not implement generate_text_completion()."
         )
 
     def set_target_language(self, language: str) -> None:
@@ -143,7 +143,7 @@ def set_target_language(self, language: str) -> None:
     def set_source_language(self, language: str) -> None:
         self.src_lang = language
 
-    def _summary_language_name(self) -> str:
+    def _memory_language_name(self) -> str:
         if getattr(self, "tgt_lang", None):
             return LANG_MAPPER.get(self.tgt_lang, self.tgt_lang)
         if getattr(self, "src_lang", None):
@@ -153,27 +153,27 @@ def _summary_language_name(self) -> str:
     def build_raw_text_prefix(self) -> str:
         return "".join(self.text_history) if self.text_history else ""
 
-    def build_summary_context(self) -> str:
-        update_prefix_summary = getattr(self.text_history_method, "update_prefix_summary", None)
-        if update_prefix_summary is None:
+    def build_memory_context(self) -> str:
+        update_prefix_memory = getattr(self.text_history_method, "update_prefix_memory", None)
+        if update_prefix_memory is None:
             return ""
-        return self.prefix_summary.strip()
+        return self.prefix_memory.strip()
 
     def _update_text_history(self, new_output: List[str]) -> int:
         previous_history = list(self.text_history) if self.text_history else []
         current_history = previous_history + new_output
         discarded_text = super()._update_text_history(new_output)
-        update_prefix_summary = getattr(self.text_history_method, "update_prefix_summary", None)
-        if discarded_text > 0 and update_prefix_summary is not None:
-            new_summary = update_prefix_summary(
-                prefix_summary=self.prefix_summary,
+        update_prefix_memory = getattr(self.text_history_method, "update_prefix_memory", None)
+        if discarded_text > 0 and update_prefix_memory is not None:
+            new_memory = update_prefix_memory(
+                prefix_memory=self.prefix_memory,
                 discarded_tokens=current_history[:discarded_text],
                 tokens_to_string=self.tokens_to_string,
-                summarize_text=self.summarize_text,
-                language_name=self._summary_language_name(),
+                generate_text_completion=self.generate_text_completion,
+                language_name=self._memory_language_name(),
             )
-            if new_summary:
-                self.prefix_summary = new_summary
+            if new_memory:
+                self.prefix_memory = new_memory
         return discarded_text
 
     def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
@@ -202,79 +202,82 @@ def _preprocess(self, waveform: np.float32) -> dict:
 
     def clear(self) -> None:
         super().clear()
-        self.prefix_summary = ""
+        self.prefix_memory = ""
 
 
-class _SummaryPrefixTextHistory:
+class _ReferenceMemoryTextHistory:
     """
-    Base summary text-history selector.
+    Base reference-memory selector.
+
+    The memory is intentionally constrained to literal high-value items rather
+    than free-form summaries, which is usually safer for translation
+    continuation.
 
     Config attributes
     -----------------
-    summary_max_new_tokens : int
-        Maximum tokens used when updating the running summary.
+    memory_max_new_tokens : int
+        Maximum tokens used when updating the running reference memory.
     """
 
     def __init__(self, config: SimpleNamespace, _bow_prefix: str):
-        self.summary_max_new_tokens = getattr(config, "summary_max_new_tokens", 32)
+        self.memory_max_new_tokens = getattr(config, "memory_max_new_tokens", 32)
 
-    def update_prefix_summary(
+    def update_prefix_memory(
             self,
-            prefix_summary: str,
+            prefix_memory: str,
             discarded_tokens: List[str],
             tokens_to_string,
-            summarize_text,
+            generate_text_completion,
             language_name: str) -> str:
         discarded_text = tokens_to_string(discarded_tokens).strip()
         if not discarded_text:
             return ""
-        if prefix_summary:
-            summary_prompt = (
-                f"Update these memory notes in {language_name}. Keep them brief and useful for "
-                f"continuing the translation. Preserve key entities, terminology, abbreviations, "
-                f"numbers, and unresolved references. Return only the updated memory notes.\n\n"
-                f"Current memory notes:\n{prefix_summary}\n\n"
+
+        if prefix_memory:
+            prompt = (
+                f"Update this compact reference memory in {language_name}. Keep only literal items "
+                f"that help continue the translation consistently. Preserve exact surface forms "
+                f"when possible. Focus on names, organizations, acronyms, technical terms, "
+                f"numbers, dates, titles, quoted phrases, URLs, code-like identifiers, and "
+                f"unresolved references. Avoid prose, paraphrases, and full-sentence summaries. "
+                f"Return only the updated reference memory.\n\n"
+                f"Current reference memory:\n{prefix_memory}\n\n"
                 f"New earlier translated context:\n{discarded_text}"
             )
         else:
-            summary_prompt = (
-                f"Summarize the following earlier translated context in {language_name}. "
-                f"Return short memory notes useful for continuing the translation. Preserve key "
-                f"entities, terminology, abbreviations, numbers, and unresolved references. "
-                f"Prefer concise notes over full prose. Return only the memory notes.\n\n"
+            prompt = (
+                f"Extract compact reference memory in {language_name} from the following earlier "
+                f"translated context. Keep only literal items that help continue the translation "
+                f"consistently. Preserve exact surface forms when possible. Focus on names, "
+                f"organizations, acronyms, technical terms, numbers, dates, titles, quoted "
+                f"phrases, URLs, code-like identifiers, and unresolved references. Avoid prose, "
+                f"paraphrases, and full-sentence summaries. Return only the reference memory.\n\n"
                 f"Earlier translated context:\n{discarded_text}"
             )
-        new_summary = summarize_text(
-            summary_prompt,
-            self.summary_max_new_tokens,
-        ).strip()
-        return new_summary
+
+        return generate_text_completion(prompt, self.memory_max_new_tokens).strip()
 
 
-class SummaryFixedWordsTextHistory(FixedWordsTextHistory, _SummaryPrefixTextHistory):
+class ReferenceMemoryFixedWordsTextHistory(FixedWordsTextHistory, _ReferenceMemoryTextHistory):
     """
-    Fixed-words text-history selector plus a DOA-only running summary prefix.
+    Fixed-words text-history selector plus a DOA-only reference memory prefix.
 
-    Config attributes
-    -----------------
-    history_words : int
-        Number of recent raw words to retain for StreamAtt alignment.
+    The raw retained history is still used for StreamAtt alignment, while older
+    discarded text is compressed into a compact list of entities, terms, and
+    numbers for the next decoder-only prompt.
     """
 
     def __init__(self, config: SimpleNamespace, bow_prefix: str):
         FixedWordsTextHistory.__init__(self, config, bow_prefix)
-        _SummaryPrefixTextHistory.__init__(self, config, bow_prefix)
+        _ReferenceMemoryTextHistory.__init__(self, config, bow_prefix)
 
 
-class SummaryPunctuationTextHistory(PunctuationTextHistory, _SummaryPrefixTextHistory):
+class ReferenceMemoryPunctuationTextHistory(PunctuationTextHistory, _ReferenceMemoryTextHistory):
     """
-    Punctuation-based text-history selector plus a DOA-only running summary prefix.
-
-    The raw retained history still follows the punctuation selector, while the
-    discarded older context is compressed into a running summary for the next
-    decoder-only prompt.
+    Punctuation-based text-history selector plus a DOA-only reference memory
+    prefix.
     """
 
     def __init__(self, config: SimpleNamespace, bow_prefix: str):
         PunctuationTextHistory.__init__(self, config, bow_prefix)
-        _SummaryPrefixTextHistory.__init__(self, config, bow_prefix)
+        _ReferenceMemoryTextHistory.__init__(self, config, bow_prefix)
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index bbc2ad0..55ee42e 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -67,14 +67,15 @@ def load_model(cls, config: SimpleNamespace) -> None:
 
     def build_prompt(self) -> str:
         filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
-        summary = self.build_summary_context()
+        memory = self.build_memory_context()
         raw_prefix = self.build_raw_text_prefix()
-        if summary:
+        if memory:
             filled_prompt = (
-                f"Background memory from earlier translated audio in {LANG_MAPPER[self.tgt_lang]}: "
-                f"{summary}\n"
-                f"Continue the existing translation in {LANG_MAPPER[self.tgt_lang]}. "
-                f"Output only the next continuation.\n\n"
+                f"Reference memory from earlier translated audio in "
+                f"{LANG_MAPPER[self.tgt_lang]}: {memory}\n"
+                f"Use this memory only to keep entities, terminology, abbreviations, numbers, "
+                f"and unresolved references consistent while continuing the translation. Output "
+                f"only the next continuation.\n\n"
                 f"{filled_prompt}"
             )
         prompt = (
@@ -91,7 +92,7 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             return_tensors="pt",
         ).to(self.device)
 
-    def summarize_text(self, prompt: str, max_new_tokens: int) -> str:
+    def generate_text_completion(self, prompt: str, max_new_tokens: int) -> str:
         inputs = self.processor(
             text=f"{self._USER_START}{prompt}{self._END_TOKEN}{self._ASST_START}",
             return_tensors="pt",
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 426c7f3..81086ac 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -99,14 +99,15 @@ def build_prompt(self) -> str:
         )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        summary = self.build_summary_context()
+        memory = self.build_memory_context()
         prompt_text = self.build_prompt()
-        if summary:
+        if memory:
             prompt_text = (
-                f"Background memory from earlier translated audio in {LANG_MAPPER[self.tgt_lang]}: "
-                f"{summary}\n"
-                f"Continue the existing translation in {LANG_MAPPER[self.tgt_lang]}. "
-                f"Output only the next continuation.\n\n"
+                f"Reference memory from earlier translated audio in "
+                f"{LANG_MAPPER[self.tgt_lang]}: {memory}\n"
+                f"Use this memory only to keep entities, terminology, abbreviations, numbers, "
+                f"and unresolved references consistent while continuing the translation. Output "
+                f"only the next continuation.\n\n"
                 f"{prompt_text}"
             )
 
@@ -144,7 +145,7 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             use_audio_in_video=True,
         ).to(self.device)
 
-    def summarize_text(self, prompt: str, max_new_tokens: int) -> str:
+    def generate_text_completion(self, prompt: str, max_new_tokens: int) -> str:
         conversation = [
             {
                 "role": "system",
@@ -155,13 +156,13 @@ def summarize_text(self, prompt: str, max_new_tokens: int) -> str:
                 "content": [{"type": "text", "text": prompt}],
             },
         ]
-        summary_prompt = self.processor.apply_chat_template(
+        text_prompt = self.processor.apply_chat_template(
             conversation,
             add_generation_prompt=True,
             tokenize=False,
         )
         inputs = self.processor(
-            text=summary_prompt,
+            text=text_prompt,
             return_tensors="pt",
             padding=True,
         ).to(self.device)

From 5dcddbcd5ecd52b682583e825bd87354f8f5b948 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 7 May 2026 10:25:33 +0200
Subject: [PATCH 073/157] Revert to simple implementation

---
 ...i4multimodal_doa_reference_fixedwords.yaml |  14 --
 ...4multimodal_doa_reference_punctuation.yaml |  13 --
 config/qwen2.5omni_3b_doa_fixedwords.yaml     |  15 ---
 .../server/speech_processors/base_doa.py      | 124 +-----------------
 .../speech_processors/phi4multimodal_doa.py   |  25 ----
 .../server/speech_processors/qwen2_5_doa.py   |  42 ------
 6 files changed, 1 insertion(+), 232 deletions(-)
 delete mode 100644 config/phi4multimodal_doa_reference_fixedwords.yaml
 delete mode 100644 config/phi4multimodal_doa_reference_punctuation.yaml
 delete mode 100644 config/qwen2.5omni_3b_doa_fixedwords.yaml

diff --git a/config/phi4multimodal_doa_reference_fixedwords.yaml b/config/phi4multimodal_doa_reference_fixedwords.yaml
deleted file mode 100644
index 75b501a..0000000
--- a/config/phi4multimodal_doa_reference_fixedwords.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_doa.ReferenceMemoryFixedWordsTextHistory"
-  history_words: __HISTORY__
-audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-cutoff_frame_num: __FRAME__
-average_attn_over_layers: True
-detokenizer_type: "hf"
-hf_model_name: "microsoft/Phi-4-multimodal-instruct"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
diff --git a/config/phi4multimodal_doa_reference_punctuation.yaml b/config/phi4multimodal_doa_reference_punctuation.yaml
deleted file mode 100644
index 66bea36..0000000
--- a/config/phi4multimodal_doa_reference_punctuation.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_doa.ReferenceMemoryPunctuationTextHistory"
-audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-cutoff_frame_num: __FRAME__
-average_attn_over_layers: True
-detokenizer_type: "hf"
-hf_model_name: "microsoft/Phi-4-multimodal-instruct"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
diff --git a/config/qwen2.5omni_3b_doa_fixedwords.yaml b/config/qwen2.5omni_3b_doa_fixedwords.yaml
deleted file mode 100644
index 621b071..0000000
--- a/config/qwen2.5omni_3b_doa_fixedwords.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
-  history_words: __HISTORY__
-audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-average_attn_over_layers: True
-cutoff_frame_num: __FRAME__
-detokenizer_type: "hf"
-hf_model_name: "Qwen/Qwen2.5-Omni-3B"
-use_video: False
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index a63a52c..ab87549 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -21,11 +21,7 @@
 import torch
 
 from simulstream.server.speech_processors import SAMPLE_RATE
-from simulstream.server.speech_processors.base_streamatt import (
-    BaseStreamAtt,
-    FixedWordsTextHistory,
-    PunctuationTextHistory,
-)
+from simulstream.server.speech_processors.base_streamatt import BaseStreamAtt
 
 logger = logging.getLogger(__name__)
 
@@ -77,7 +73,6 @@ def __init__(self, config: SimpleNamespace):
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
-        self.prefix_memory = ""
 
     @property
     def audio_max_len(self) -> int:
@@ -132,50 +127,15 @@ def tokens_to_string(self, tokens: List[str]) -> str:
         """Convert a list of decoded tokens to a plain output string."""
         ...
 
-    def generate_text_completion(self, prompt: str, max_new_tokens: int) -> str:
-        raise NotImplementedError(
-            f"{self.__class__.__name__} does not implement generate_text_completion()."
-        )
-
     def set_target_language(self, language: str) -> None:
         self.tgt_lang = language
 
     def set_source_language(self, language: str) -> None:
         self.src_lang = language
 
-    def _memory_language_name(self) -> str:
-        if getattr(self, "tgt_lang", None):
-            return LANG_MAPPER.get(self.tgt_lang, self.tgt_lang)
-        if getattr(self, "src_lang", None):
-            return LANG_MAPPER.get(self.src_lang, self.src_lang)
-        return "the same language as the context"
-
     def build_raw_text_prefix(self) -> str:
         return "".join(self.text_history) if self.text_history else ""
 
-    def build_memory_context(self) -> str:
-        update_prefix_memory = getattr(self.text_history_method, "update_prefix_memory", None)
-        if update_prefix_memory is None:
-            return ""
-        return self.prefix_memory.strip()
-
-    def _update_text_history(self, new_output: List[str]) -> int:
-        previous_history = list(self.text_history) if self.text_history else []
-        current_history = previous_history + new_output
-        discarded_text = super()._update_text_history(new_output)
-        update_prefix_memory = getattr(self.text_history_method, "update_prefix_memory", None)
-        if discarded_text > 0 and update_prefix_memory is not None:
-            new_memory = update_prefix_memory(
-                prefix_memory=self.prefix_memory,
-                discarded_tokens=current_history[:discarded_text],
-                tokens_to_string=self.tokens_to_string,
-                generate_text_completion=self.generate_text_completion,
-                language_name=self._memory_language_name(),
-            )
-            if new_memory:
-                self.prefix_memory = new_memory
-        return discarded_text
-
     def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
         if self.average_attn_over_layers:
             return torch.stack(
@@ -199,85 +159,3 @@ def _preprocess(self, waveform: np.float32) -> dict:
             self.audio_history = self.audio_history[-self.audio_max_len:]
 
         return self.build_processor_inputs(self.audio_history)
-
-    def clear(self) -> None:
-        super().clear()
-        self.prefix_memory = ""
-
-
-class _ReferenceMemoryTextHistory:
-    """
-    Base reference-memory selector.
-
-    The memory is intentionally constrained to literal high-value items rather
-    than free-form summaries, which is usually safer for translation
-    continuation.
-
-    Config attributes
-    -----------------
-    memory_max_new_tokens : int
-        Maximum tokens used when updating the running reference memory.
-    """
-
-    def __init__(self, config: SimpleNamespace, _bow_prefix: str):
-        self.memory_max_new_tokens = getattr(config, "memory_max_new_tokens", 32)
-
-    def update_prefix_memory(
-            self,
-            prefix_memory: str,
-            discarded_tokens: List[str],
-            tokens_to_string,
-            generate_text_completion,
-            language_name: str) -> str:
-        discarded_text = tokens_to_string(discarded_tokens).strip()
-        if not discarded_text:
-            return ""
-
-        if prefix_memory:
-            prompt = (
-                f"Update this compact reference memory in {language_name}. Keep only literal items "
-                f"that help continue the translation consistently. Preserve exact surface forms "
-                f"when possible. Focus on names, organizations, acronyms, technical terms, "
-                f"numbers, dates, titles, quoted phrases, URLs, code-like identifiers, and "
-                f"unresolved references. Avoid prose, paraphrases, and full-sentence summaries. "
-                f"Return only the updated reference memory.\n\n"
-                f"Current reference memory:\n{prefix_memory}\n\n"
-                f"New earlier translated context:\n{discarded_text}"
-            )
-        else:
-            prompt = (
-                f"Extract compact reference memory in {language_name} from the following earlier "
-                f"translated context. Keep only literal items that help continue the translation "
-                f"consistently. Preserve exact surface forms when possible. Focus on names, "
-                f"organizations, acronyms, technical terms, numbers, dates, titles, quoted "
-                f"phrases, URLs, code-like identifiers, and unresolved references. Avoid prose, "
-                f"paraphrases, and full-sentence summaries. Return only the reference memory.\n\n"
-                f"Earlier translated context:\n{discarded_text}"
-            )
-
-        return generate_text_completion(prompt, self.memory_max_new_tokens).strip()
-
-
-class ReferenceMemoryFixedWordsTextHistory(FixedWordsTextHistory, _ReferenceMemoryTextHistory):
-    """
-    Fixed-words text-history selector plus a DOA-only reference memory prefix.
-
-    The raw retained history is still used for StreamAtt alignment, while older
-    discarded text is compressed into a compact list of entities, terms, and
-    numbers for the next decoder-only prompt.
-    """
-
-    def __init__(self, config: SimpleNamespace, bow_prefix: str):
-        FixedWordsTextHistory.__init__(self, config, bow_prefix)
-        _ReferenceMemoryTextHistory.__init__(self, config, bow_prefix)
-
-
-class ReferenceMemoryPunctuationTextHistory(PunctuationTextHistory, _ReferenceMemoryTextHistory):
-    """
-    Punctuation-based text-history selector plus a DOA-only reference memory
-    prefix.
-    """
-
-    def __init__(self, config: SimpleNamespace, bow_prefix: str):
-        PunctuationTextHistory.__init__(self, config, bow_prefix)
-        _ReferenceMemoryTextHistory.__init__(self, config, bow_prefix)
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 55ee42e..72363a3 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -67,17 +67,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
 
     def build_prompt(self) -> str:
         filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
-        memory = self.build_memory_context()
         raw_prefix = self.build_raw_text_prefix()
-        if memory:
-            filled_prompt = (
-                f"Reference memory from earlier translated audio in "
-                f"{LANG_MAPPER[self.tgt_lang]}: {memory}\n"
-                f"Use this memory only to keep entities, terminology, abbreviations, numbers, "
-                f"and unresolved references consistent while continuing the translation. Output "
-                f"only the next continuation.\n\n"
-                f"{filled_prompt}"
-            )
         prompt = (
             f"{self._USER_START}{self._AUDIO_TOKEN}"
             f"{filled_prompt}{self._END_TOKEN}"
@@ -92,21 +82,6 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             return_tensors="pt",
         ).to(self.device)
 
-    def generate_text_completion(self, prompt: str, max_new_tokens: int) -> str:
-        inputs = self.processor(
-            text=f"{self._USER_START}{prompt}{self._END_TOKEN}{self._ASST_START}",
-            return_tensors="pt",
-        ).to(self.device)
-        output = self.model.generate(
-            **inputs,
-            max_new_tokens=max_new_tokens,
-            generation_config=self.generation_config,
-            num_logits_to_keep=1,
-            do_sample=False,
-        )
-        new_ids = output[:, inputs["input_ids"].shape[1]:]
-        return self.processor.tokenizer.decode(new_ids[0], skip_special_tokens=True).strip()
-
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         """
         Run greedy generation and build the proxy cross-attention matrix.
diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 81086ac..c808dc1 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -99,17 +99,7 @@ def build_prompt(self) -> str:
         )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        memory = self.build_memory_context()
         prompt_text = self.build_prompt()
-        if memory:
-            prompt_text = (
-                f"Reference memory from earlier translated audio in "
-                f"{LANG_MAPPER[self.tgt_lang]}: {memory}\n"
-                f"Use this memory only to keep entities, terminology, abbreviations, numbers, "
-                f"and unresolved references consistent while continuing the translation. Output "
-                f"only the next continuation.\n\n"
-                f"{prompt_text}"
-            )
 
         conversation = [
             {
@@ -145,38 +135,6 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             use_audio_in_video=True,
         ).to(self.device)
 
-    def generate_text_completion(self, prompt: str, max_new_tokens: int) -> str:
-        conversation = [
-            {
-                "role": "system",
-                "content": [{"type": "text", "text": self.SYSTEM_PROMPT}],
-            },
-            {
-                "role": "user",
-                "content": [{"type": "text", "text": prompt}],
-            },
-        ]
-        text_prompt = self.processor.apply_chat_template(
-            conversation,
-            add_generation_prompt=True,
-            tokenize=False,
-        )
-        inputs = self.processor(
-            text=text_prompt,
-            return_tensors="pt",
-            padding=True,
-        ).to(self.device)
-        output = self.model.generate(
-            **inputs,
-            return_audio=False,
-            thinker_max_new_tokens=max_new_tokens,
-            thinker_do_sample=False,
-        )
-        if isinstance(output, tuple):
-            output = output[0]
-        new_ids = output[:, inputs["input_ids"].shape[1]:]
-        return self.processor.tokenizer.decode(new_ids[0], skip_special_tokens=True).strip()
-
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
         if audio_positions.numel() > 0:

From acbc5e19a8f7a4004e7e8416e45d0237d0988fd5 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 8 May 2026 18:40:34 +0200
Subject: [PATCH 074/157] Add head analysis

---
 config/phi4multimodal_doa_fixedwords.yaml     |  1 +
 config/phi4multimodal_doa_punctuation.yaml    |  1 +
 config/qwen2.5omni_3b_doa_punctuation.yaml    |  1 +
 config/qwen2.5omni_7b_doa_punctuation.yaml    |  1 +
 .../server/speech_processors/base_doa.py      | 36 +++++++++++++++++--
 5 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/config/phi4multimodal_doa_fixedwords.yaml b/config/phi4multimodal_doa_fixedwords.yaml
index 3a2cb83..78ace66 100644
--- a/config/phi4multimodal_doa_fixedwords.yaml
+++ b/config/phi4multimodal_doa_fixedwords.yaml
@@ -6,6 +6,7 @@ audio_history_max_duration: 180  # Maximum length for the audio buffer, in secon
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__
+attn_head: null  # Optional specific head; null means average over heads
 cutoff_frame_num: __FRAME__
 average_attn_over_layers: True
 detokenizer_type: "hf"
diff --git a/config/phi4multimodal_doa_punctuation.yaml b/config/phi4multimodal_doa_punctuation.yaml
index bb22119..804035f 100644
--- a/config/phi4multimodal_doa_punctuation.yaml
+++ b/config/phi4multimodal_doa_punctuation.yaml
@@ -5,6 +5,7 @@ audio_history_max_duration: 180  # Maximum length for the audio buffer, in secon
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__
+attn_head: null  # Optional specific head; null means average over heads
 cutoff_frame_num: __FRAME__
 average_attn_over_layers: True
 detokenizer_type: "hf"
diff --git a/config/qwen2.5omni_3b_doa_punctuation.yaml b/config/qwen2.5omni_3b_doa_punctuation.yaml
index aeaaa58..dec535d 100644
--- a/config/qwen2.5omni_3b_doa_punctuation.yaml
+++ b/config/qwen2.5omni_3b_doa_punctuation.yaml
@@ -5,6 +5,7 @@ audio_history_max_duration: 120  # Maximum length for the audio buffer, in secon
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__
+attn_head: null  # Optional specific head; null means average over heads
 average_attn_over_layers: True
 cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
diff --git a/config/qwen2.5omni_7b_doa_punctuation.yaml b/config/qwen2.5omni_7b_doa_punctuation.yaml
index ed5cc42..77181a3 100644
--- a/config/qwen2.5omni_7b_doa_punctuation.yaml
+++ b/config/qwen2.5omni_7b_doa_punctuation.yaml
@@ -5,6 +5,7 @@ audio_history_max_duration: 90  # Maximum length for the audio buffer, in second
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__
+attn_head: null  # Optional specific head; null means average over heads
 average_attn_over_layers: True
 cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index ab87549..3274968 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -55,6 +55,11 @@ class DecoderOnlyAttention(BaseStreamAtt):
         All fields from :class:`BaseStreamAtt`, plus:
         attn_layer : int
             Layer from which to extract attention scores. Default: ``0``.
+        attn_head : int | None
+            Attention head to use. If ``None``, attention scores are averaged
+            over all heads. If set together with
+            ``average_attn_over_layers=True``, the selected head is averaged
+            across layers. Default: ``None``.
         average_attn_over_layers : bool
             Whether to average attention over all decoder layers instead of
             using the single layer selected by ``attn_layer``.
@@ -64,11 +69,25 @@ class DecoderOnlyAttention(BaseStreamAtt):
             Default: ``180`` (seconds).
         max_new_tokens : int
             Maximum tokens to generate per chunk.  Default: ``32``.
+
+    Supported attention-selection modes
+    -----------------------------------
+    - ``attn_head=None`` and ``average_attn_over_layers=True``:
+      average across layers and heads.
+    - ``attn_head=None`` and ``average_attn_over_layers=False``:
+      average across heads within ``attn_layer``.
+    - ``attn_head=<int>`` and ``average_attn_over_layers=True``:
+      average across layers within the selected head.
+
+    An additional mode is also supported for completeness:
+    - ``attn_head=<int>`` and ``average_attn_over_layers=False``:
+      use the selected head within ``attn_layer``.
     """
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
         self.cross_attn_layer = getattr(self.config, "attn_layer", 0)
+        self.cross_attn_head = getattr(self.config, "attn_head", None)
         self.average_attn_over_layers = getattr(self.config, "average_attn_over_layers", False)
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -136,13 +155,26 @@ def set_source_language(self, language: str) -> None:
     def build_raw_text_prefix(self) -> str:
         return "".join(self.text_history) if self.text_history else ""
 
+    def _select_attn_from_layer(self, layer_attn: torch.Tensor) -> torch.Tensor:
+        if self.cross_attn_head is None:
+            # Default behavior: average over all heads for this layer.
+            return layer_attn[0].mean(dim=0)
+
+        num_heads = layer_attn.shape[1]
+        if self.cross_attn_head < 0 or self.cross_attn_head >= num_heads:
+            raise ValueError(
+                f"Invalid attn_head={self.cross_attn_head}. Layer has {num_heads} heads."
+            )
+        return layer_attn[0, self.cross_attn_head]
+
     def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
         if self.average_attn_over_layers:
+            # Average the per-layer attention view selected by _select_attn_from_layer.
             return torch.stack(
-                [layer_attn[0].mean(dim=0) for layer_attn in step_attn],
+                [self._select_attn_from_layer(layer_attn) for layer_attn in step_attn],
                 dim=0,
             ).mean(dim=0)
-        return step_attn[self.cross_attn_layer][0].mean(dim=0)
+        return self._select_attn_from_layer(step_attn[self.cross_attn_layer])
 
     def _preprocess(self, waveform: np.float32) -> dict:
         """

From 67dfd7376c276ca30a8097481bcb5b7c28aa9a3e Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 15 May 2026 17:58:13 +0200
Subject: [PATCH 075/157] Try different prompt

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index c808dc1..e3ff3ab 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -92,11 +92,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
         cls.model.eval()
 
     def build_prompt(self) -> str:
-        return (
-            TEMPLATED_SPEECH_PROMPT
-            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
-            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
-        )
+        return f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         prompt_text = self.build_prompt()

From 0506a9b22cfa16e5beea46ea62c94d716a65ae86 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 15 May 2026 19:02:52 +0200
Subject: [PATCH 076/157] Try different sys prompt

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index e3ff3ab..d22a436 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -58,8 +58,12 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     AUDIO_START_TOKEN_ID = 151647
     AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
-        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
-        "perceiving auditory and visual inputs, as well as generating text and speech."
+        "You are a speech translation system. "
+        "Translate the audio input into the target language. "
+        "Output only the translation. "
+        "Do not ask questions, do not add commentary, do not simulate a conversation, "
+        "do not write 'Human:', 'Assistant:', or any dialogue markers, including newlines. "
+        "If the audio is unclear or incomplete, output only what you can translate and stop."
     )
 
     def __init__(self, config: SimpleNamespace):

From d8133a9bb6e73493e89fa7c568f64160545586af Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 15 May 2026 20:58:16 +0200
Subject: [PATCH 077/157] debug

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index d22a436..99f55c4 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -196,6 +196,9 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
+        print("prefill:", output.attentions[0][0].shape)  # layer 0, prefill
+        if len(output.attentions) > 1:
+            print("step 1:", output.attentions[1][0].shape)  # layer 0, first decode step
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]

From 9448e0d5c341c1f4d9854a8fbb4840ab2d60e089 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 15 May 2026 21:18:09 +0200
Subject: [PATCH 078/157] debug

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 99f55c4..7acd433 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -186,6 +186,9 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
             for token_id in new_ids[0]
         ]
+        # Add this:
+        print("new_ids:", new_ids[0].tolist())
+        print("new_tokens:", new_tokens)
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
@@ -196,9 +199,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
-        print("prefill:", output.attentions[0][0].shape)  # layer 0, prefill
-        if len(output.attentions) > 1:
-            print("step 1:", output.attentions[1][0].shape)  # layer 0, first decode step
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]

From 9904d4b78d05c9865d230f01610a7993ffada532 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 15 May 2026 21:22:38 +0200
Subject: [PATCH 079/157] debug

---
 .../server/speech_processors/qwen2_5_doa.py       | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 7acd433..f1816a5 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -182,13 +182,14 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        new_tokens = [
-            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in new_ids[0]
-        ]
-        # Add this:
-        print("new_ids:", new_ids[0].tolist())
-        print("new_tokens:", new_tokens)
+        eos_id = self.processor.tokenizer.eos_token_id
+        new_tokens = []
+        for token_id in new_ids[0]:
+            if token_id.item() == eos_id:
+                break
+            new_tokens.append(
+                self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            )
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0

From 7e976f7779b250c3167d7ae2f52a41b815132ff6 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 15 May 2026 22:31:43 +0200
Subject: [PATCH 080/157] debug

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index f1816a5..cf69966 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -182,6 +182,9 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
+        for token_id in new_ids[0].tolist():
+            raw = self.processor.tokenizer.decode([token_id], skip_special_tokens=False)
+            print(f"{token_id}: {repr(raw)}")
         eos_id = self.processor.tokenizer.eos_token_id
         new_tokens = []
         for token_id in new_ids[0]:

From 41cb2792b2f832d4eddff907a55b1c3fc45afa9f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 15 May 2026 22:36:47 +0200
Subject: [PATCH 081/157] debug

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index cf69966..3ac4a86 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -177,18 +177,16 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
             temperature=self.temperature,
+            eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
         )
         if isinstance(output, tuple):
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        for token_id in new_ids[0].tolist():
-            raw = self.processor.tokenizer.decode([token_id], skip_special_tokens=False)
-            print(f"{token_id}: {repr(raw)}")
-        eos_id = self.processor.tokenizer.eos_token_id
+        stop_ids = {151643, 151645}
         new_tokens = []
         for token_id in new_ids[0]:
-            if token_id.item() == eos_id:
+            if token_id.item() in stop_ids:
                 break
             new_tokens.append(
                 self.processor.tokenizer.decode([token_id], skip_special_tokens=True)

From b04d775a61eb1f7feaf3cb641155c6f481e7520c Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 16 May 2026 10:16:26 +0200
Subject: [PATCH 082/157] debug

---
 .../server/speech_processors/qwen2_5_doa.py     | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 3ac4a86..275c626 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -183,11 +183,19 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        stop_ids = {151643, 151645}
+        stop_ids = {151643}  # <|endoftext|>
+        strip_ids = {151645}  # <|im_end|>
+
+        keep_mask = []
         new_tokens = []
         for token_id in new_ids[0]:
-            if token_id.item() in stop_ids:
+            tid = token_id.item()
+            if tid in stop_ids:
                 break
+            if tid in strip_ids:
+                keep_mask.append(False)
+                continue
+            keep_mask.append(True)
             new_tokens.append(
                 self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
             )
@@ -208,6 +216,11 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+
+        # Align with filtered tokens
+        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
+        new_attn = new_attn[keep_tensor]
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
         cross_attn = self.normalize_attn(cross_attn)

From fb91d6e229dd13a2759f64562136de8978f11aec Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 16 May 2026 10:44:10 +0200
Subject: [PATCH 083/157] debug

---
 .../server/speech_processors/qwen2_5_doa.py   | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 275c626..bf50595 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -177,12 +177,13 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
             temperature=self.temperature,
-            eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
         )
         if isinstance(output, tuple):
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
+
+        # Decode tokens, stopping at <|endoftext|> and skipping <|im_end|>
         stop_ids = {151643}  # <|endoftext|>
         strip_ids = {151645}  # <|im_end|>
 
@@ -200,6 +201,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
                 self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
             )
 
+        # Build proxy cross-attention
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
         if prefix_len > 0:
@@ -207,19 +209,20 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         else:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
 
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
+        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
+
+        first_new_row = prefill_attn[-1:, audio_positions] if keep_mask else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
+
         new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:]
+            self.mean_attn_over_heads_and_selected_layers(step_attn)
+            .squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:len(keep_mask) + 1]
         ]
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
-        # Align with filtered tokens
-        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
         new_attn = new_attn[keep_tensor]
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)

From 5d68d6a86d56dfe7ca42940a6c756d50d2acbf2d Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 16 May 2026 11:08:19 +0200
Subject: [PATCH 084/157] debug

---
 .../server/speech_processors/qwen2_5_doa.py   | 33 +++++++++++--------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index bf50595..c270ebc 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -209,21 +209,26 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         else:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
 
-        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
-
-        first_new_row = prefill_attn[-1:, audio_positions] if keep_mask else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
-
-        new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn)
-            .squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:len(keep_mask) + 1]
-        ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
+        # Build all new token attention rows uniformly:
+        # - index 0 = prefill last row (predicts first new token)
+        # - index 1..n = decode steps from output.attentions[1:]
+        # keep_mask has exactly len(keep_mask) entries, one per non-EOS token in new_ids
+        n_generated = len(keep_mask)  # tokens before EOS, including <|im_end|> ones
+        all_new_rows = []
+        if n_generated > 0:
+            all_new_rows.append(prefill_attn[-1:, audio_positions])  # first token row
+        for step_attn in output.attentions[1:n_generated]:
+            all_new_rows.append(
+                self.mean_attn_over_heads_and_selected_layers(step_attn)
+                .squeeze(0)[audio_positions].unsqueeze(0)
+            )
 
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-        new_attn = new_attn[keep_tensor]
+        if all_new_rows:
+            new_attn = torch.cat(all_new_rows, dim=0)  # (n_generated, audio_len)
+            keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
+            new_attn = new_attn[keep_tensor]  # (n_kept, audio_len)
+        else:
+            new_attn = torch.zeros(0, max(audio_len, 1), device=self.device)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
         cross_attn = self.normalize_attn(cross_attn)

From 8ff9bd12df705f087dd9e1715afcc5d8149246de Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:31:37 +0200
Subject: [PATCH 085/157] Revert "debug"

This reverts commit 5d68d6a86d56dfe7ca42940a6c756d50d2acbf2d.
---
 .../server/speech_processors/qwen2_5_doa.py   | 33 ++++++++-----------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index c270ebc..bf50595 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -209,26 +209,21 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         else:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
 
-        # Build all new token attention rows uniformly:
-        # - index 0 = prefill last row (predicts first new token)
-        # - index 1..n = decode steps from output.attentions[1:]
-        # keep_mask has exactly len(keep_mask) entries, one per non-EOS token in new_ids
-        n_generated = len(keep_mask)  # tokens before EOS, including <|im_end|> ones
-        all_new_rows = []
-        if n_generated > 0:
-            all_new_rows.append(prefill_attn[-1:, audio_positions])  # first token row
-        for step_attn in output.attentions[1:n_generated]:
-            all_new_rows.append(
-                self.mean_attn_over_heads_and_selected_layers(step_attn)
-                .squeeze(0)[audio_positions].unsqueeze(0)
-            )
+        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
 
-        if all_new_rows:
-            new_attn = torch.cat(all_new_rows, dim=0)  # (n_generated, audio_len)
-            keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
-            new_attn = new_attn[keep_tensor]  # (n_kept, audio_len)
-        else:
-            new_attn = torch.zeros(0, max(audio_len, 1), device=self.device)
+        first_new_row = prefill_attn[-1:, audio_positions] if keep_mask else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
+
+        new_rows = [
+            self.mean_attn_over_heads_and_selected_layers(step_attn)
+            .squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:len(keep_mask) + 1]
+        ]
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
+
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+        new_attn = new_attn[keep_tensor]
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
         cross_attn = self.normalize_attn(cross_attn)

From ed17179e322ac67ceaa460544a3b0315aaa2b203 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:31:41 +0200
Subject: [PATCH 086/157] Revert "debug"

This reverts commit fb91d6e229dd13a2759f64562136de8978f11aec.
---
 .../server/speech_processors/qwen2_5_doa.py   | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index bf50595..275c626 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -177,13 +177,12 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
             temperature=self.temperature,
+            eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
         )
         if isinstance(output, tuple):
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-
-        # Decode tokens, stopping at <|endoftext|> and skipping <|im_end|>
         stop_ids = {151643}  # <|endoftext|>
         strip_ids = {151645}  # <|im_end|>
 
@@ -201,7 +200,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
                 self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
             )
 
-        # Build proxy cross-attention
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
         if prefix_len > 0:
@@ -209,20 +207,19 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         else:
             prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
 
-        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
-
-        first_new_row = prefill_attn[-1:, audio_positions] if keep_mask else \
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
-
         new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn)
-            .squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:len(keep_mask) + 1]
+            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:]
         ]
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
-
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+
+        # Align with filtered tokens
+        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
         new_attn = new_attn[keep_tensor]
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)

From 421a2f8a43c47b5270e8048c81440533805d834c Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:31:47 +0200
Subject: [PATCH 087/157] Revert "debug"

This reverts commit b04d775a61eb1f7feaf3cb641155c6f481e7520c.
---
 .../server/speech_processors/qwen2_5_doa.py     | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 275c626..3ac4a86 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -183,19 +183,11 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        stop_ids = {151643}  # <|endoftext|>
-        strip_ids = {151645}  # <|im_end|>
-
-        keep_mask = []
+        stop_ids = {151643, 151645}
         new_tokens = []
         for token_id in new_ids[0]:
-            tid = token_id.item()
-            if tid in stop_ids:
+            if token_id.item() in stop_ids:
                 break
-            if tid in strip_ids:
-                keep_mask.append(False)
-                continue
-            keep_mask.append(True)
             new_tokens.append(
                 self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
             )
@@ -216,11 +208,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-
-        # Align with filtered tokens
-        keep_tensor = torch.tensor(keep_mask, dtype=torch.bool, device=self.device)
-        new_attn = new_attn[keep_tensor]
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
         cross_attn = self.normalize_attn(cross_attn)

From 6f3f6f4a6e60a25c1b677e3aefe21d851147f85b Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:31:51 +0200
Subject: [PATCH 088/157] Revert "debug"

This reverts commit 41cb2792b2f832d4eddff907a55b1c3fc45afa9f.
---
 simulstream/server/speech_processors/qwen2_5_doa.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 3ac4a86..cf69966 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -177,16 +177,18 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
             temperature=self.temperature,
-            eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
         )
         if isinstance(output, tuple):
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        stop_ids = {151643, 151645}
+        for token_id in new_ids[0].tolist():
+            raw = self.processor.tokenizer.decode([token_id], skip_special_tokens=False)
+            print(f"{token_id}: {repr(raw)}")
+        eos_id = self.processor.tokenizer.eos_token_id
         new_tokens = []
         for token_id in new_ids[0]:
-            if token_id.item() in stop_ids:
+            if token_id.item() == eos_id:
                 break
             new_tokens.append(
                 self.processor.tokenizer.decode([token_id], skip_special_tokens=True)

From b94cb83d628f959c89cdd363f1e931d24f4a227e Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:31:54 +0200
Subject: [PATCH 089/157] Revert "debug"

This reverts commit 7e976f7779b250c3167d7ae2f52a41b815132ff6.
---
 simulstream/server/speech_processors/qwen2_5_doa.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index cf69966..f1816a5 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -182,9 +182,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        for token_id in new_ids[0].tolist():
-            raw = self.processor.tokenizer.decode([token_id], skip_special_tokens=False)
-            print(f"{token_id}: {repr(raw)}")
         eos_id = self.processor.tokenizer.eos_token_id
         new_tokens = []
         for token_id in new_ids[0]:

From 4a7414b69ec94c51bf59be4ab1002d1000f721ad Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:31:58 +0200
Subject: [PATCH 090/157] Revert "debug"

This reverts commit 9904d4b78d05c9865d230f01610a7993ffada532.
---
 .../server/speech_processors/qwen2_5_doa.py       | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index f1816a5..7acd433 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -182,14 +182,13 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        eos_id = self.processor.tokenizer.eos_token_id
-        new_tokens = []
-        for token_id in new_ids[0]:
-            if token_id.item() == eos_id:
-                break
-            new_tokens.append(
-                self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            )
+        new_tokens = [
+            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in new_ids[0]
+        ]
+        # Add this:
+        print("new_ids:", new_ids[0].tolist())
+        print("new_tokens:", new_tokens)
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0

From deec3a81bbfae888e8ff3064816c449d28c73f31 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:32:02 +0200
Subject: [PATCH 091/157] Revert "debug"

This reverts commit 9448e0d5c341c1f4d9854a8fbb4840ab2d60e089.
---
 simulstream/server/speech_processors/qwen2_5_doa.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 7acd433..99f55c4 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -186,9 +186,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
             for token_id in new_ids[0]
         ]
-        # Add this:
-        print("new_ids:", new_ids[0].tolist())
-        print("new_tokens:", new_tokens)
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
@@ -199,6 +196,9 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
+        print("prefill:", output.attentions[0][0].shape)  # layer 0, prefill
+        if len(output.attentions) > 1:
+            print("step 1:", output.attentions[1][0].shape)  # layer 0, first decode step
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]

From 525f20bd965ffb9f4f54697228029e127839abc8 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 11:32:06 +0200
Subject: [PATCH 092/157] Revert "debug"

This reverts commit d8133a9bb6e73493e89fa7c568f64160545586af.
---
 simulstream/server/speech_processors/qwen2_5_doa.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 99f55c4..d22a436 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -196,9 +196,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
-        print("prefill:", output.attentions[0][0].shape)  # layer 0, prefill
-        if len(output.attentions) > 1:
-            print("step 1:", output.attentions[1][0].shape)  # layer 0, first decode step
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]

From bb047fb1da7f829a139bde214a81af3873ce9fe8 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 12:10:21 +0200
Subject: [PATCH 093/157] debug

---
 .../server/speech_processors/qwen2_5_doa.py      | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index d22a436..2d6751a 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-import logging
 from types import SimpleNamespace
 from typing import List, Tuple
 
@@ -34,9 +33,6 @@
 set_seed(42)
 
 
-logger = logging.getLogger(__name__)
-
-
 class Qwen2_5OmniDOA(DecoderOnlyAttention):
     """
     Decoder-Only Attention agent for ``Qwen/Qwen2.5-Omni-*``.
@@ -57,6 +53,7 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     AUDIO_TOKEN_INDEX = 151646
     AUDIO_START_TOKEN_ID = 151647
     AUDIO_END_TOKEN_ID = 151648
+    THINKER_EOS_TOKEN_IDS = [151643, 151645]  # <|endoftext|>, <|im_end|>
     SYSTEM_PROMPT = (
         "You are a speech translation system. "
         "Translate the audio input into the target language. "
@@ -171,6 +168,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             use_audio_in_video=True,
             return_audio=False,
             thinker_max_new_tokens=self.max_new_tokens,
+            thinker_eos_token_id=self.THINKER_EOS_TOKEN_IDS,
             thinker_repetition_penalty=self.repetition_penalty,
             thinker_no_repeat_ngram_size=self.no_repeat_ngram_size,
             thinker_output_attentions=True,
@@ -182,10 +180,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        new_tokens = [
-            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in new_ids[0]
-        ]
+        generated_ids = new_ids[0].tolist()
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
@@ -204,6 +199,11 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
+        new_tokens = [
+            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in generated_ids
+        ]
+
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
         cross_attn = self.normalize_attn(cross_attn)
         return new_tokens, cross_attn

From 8247ecc284e13df5b76464b067a69aa34a5dd383 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 12:12:35 +0200
Subject: [PATCH 094/157] Revert "debug"

This reverts commit bb047fb1da7f829a139bde214a81af3873ce9fe8.
---
 .../server/speech_processors/qwen2_5_doa.py      | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 2d6751a..d22a436 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
+import logging
 from types import SimpleNamespace
 from typing import List, Tuple
 
@@ -33,6 +34,9 @@
 set_seed(42)
 
 
+logger = logging.getLogger(__name__)
+
+
 class Qwen2_5OmniDOA(DecoderOnlyAttention):
     """
     Decoder-Only Attention agent for ``Qwen/Qwen2.5-Omni-*``.
@@ -53,7 +57,6 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     AUDIO_TOKEN_INDEX = 151646
     AUDIO_START_TOKEN_ID = 151647
     AUDIO_END_TOKEN_ID = 151648
-    THINKER_EOS_TOKEN_IDS = [151643, 151645]  # <|endoftext|>, <|im_end|>
     SYSTEM_PROMPT = (
         "You are a speech translation system. "
         "Translate the audio input into the target language. "
@@ -168,7 +171,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             use_audio_in_video=True,
             return_audio=False,
             thinker_max_new_tokens=self.max_new_tokens,
-            thinker_eos_token_id=self.THINKER_EOS_TOKEN_IDS,
             thinker_repetition_penalty=self.repetition_penalty,
             thinker_no_repeat_ngram_size=self.no_repeat_ngram_size,
             thinker_output_attentions=True,
@@ -180,7 +182,10 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             output = output[0]
 
         new_ids = output.sequences[:, input_len:]
-        generated_ids = new_ids[0].tolist()
+        new_tokens = [
+            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in new_ids[0]
+        ]
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
@@ -199,11 +204,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
-        new_tokens = [
-            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in generated_ids
-        ]
-
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
         cross_attn = self.normalize_attn(cross_attn)
         return new_tokens, cross_attn

From fc40392d4d96fc4c373c28adb5b0c98fad520ad6 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 12:13:29 +0200
Subject: [PATCH 095/157] debug

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index d22a436..a598404 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -176,6 +176,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
+            thinker_eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
             temperature=self.temperature,
         )
         if isinstance(output, tuple):

From c1503487a5094a6305fa0f338a9a52f7a1766bf1 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 13:49:18 +0200
Subject: [PATCH 096/157] Revert "Try different sys prompt"

This reverts commit 0506a9b22cfa16e5beea46ea62c94d716a65ae86.
---
 simulstream/server/speech_processors/qwen2_5_doa.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index a598404..030d015 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -58,12 +58,8 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     AUDIO_START_TOKEN_ID = 151647
     AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
-        "You are a speech translation system. "
-        "Translate the audio input into the target language. "
-        "Output only the translation. "
-        "Do not ask questions, do not add commentary, do not simulate a conversation, "
-        "do not write 'Human:', 'Assistant:', or any dialogue markers, including newlines. "
-        "If the audio is unclear or incomplete, output only what you can translate and stop."
+        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
+        "perceiving auditory and visual inputs, as well as generating text and speech."
     )
 
     def __init__(self, config: SimpleNamespace):

From f6ff53dd7ea54e7ec35051b3e4add5cc8fddd415 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 17 May 2026 13:57:03 +0200
Subject: [PATCH 097/157] revert

---
 simulstream/server/speech_processors/qwen2_5_doa.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwen2_5_doa.py
index 030d015..0831cd5 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwen2_5_doa.py
@@ -58,8 +58,12 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     AUDIO_START_TOKEN_ID = 151647
     AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
-        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
-        "perceiving auditory and visual inputs, as well as generating text and speech."
+        "You are a speech translation system. "
+        "Translate the audio input into the target language. "
+        "Output only the translation. "
+        "Do not ask questions, do not add commentary, do not simulate a conversation, "
+        "do not write 'Human:', 'Assistant:', or any dialogue markers, including newlines. "
+        "If the audio is unclear or incomplete, output only what you can translate and stop."
     )
 
     def __init__(self, config: SimpleNamespace):
@@ -172,7 +176,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
-            thinker_eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
+            #thinker_eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
             temperature=self.temperature,
         )
         if isinstance(output, tuple):

From a81af66c6ebd3b1341c7c3b18e4ad91232057c61 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 11:39:46 +0200
Subject: [PATCH 098/157] Try new Qwen3Omni

---
 .../{qwen2_5_doa.py => qwenomni_doa.py}       | 43 +++++++++----------
 1 file changed, 21 insertions(+), 22 deletions(-)
 rename simulstream/server/speech_processors/{qwen2_5_doa.py => qwenomni_doa.py} (81%)

diff --git a/simulstream/server/speech_processors/qwen2_5_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
similarity index 81%
rename from simulstream/server/speech_processors/qwen2_5_doa.py
rename to simulstream/server/speech_processors/qwenomni_doa.py
index 0831cd5..4560ba8 100644
--- a/simulstream/server/speech_processors/qwen2_5_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -20,7 +20,7 @@
 import torch
 
 from qwen_omni_utils import process_mm_info
-from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor
+from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor
 
 from simulstream.server.speech_processors import SAMPLE_RATE, class_load
 from simulstream.server.speech_processors.base_doa import (
@@ -37,15 +37,14 @@
 logger = logging.getLogger(__name__)
 
 
-class Qwen2_5OmniDOA(DecoderOnlyAttention):
+class Qwen3OmniDOA(DecoderOnlyAttention):
     """
-    Decoder-Only Attention agent for ``Qwen/Qwen2.5-Omni-*``.
+    Decoder-Only Attention agent for ``Qwen/Qwen3-Omni-*``.
 
     Extra config fields
     -------------------
     hf_model_name : str
-        Default: ``"Qwen/Qwen2.5-Omni-7B"``.
-        ``"Qwen/Qwen2.5-Omni-3B"`` is also supported.
+        Default: ``"Qwen/Qwen3-Omni-30B-A3B-Instruct"``.
     repetition_penalty : float
         Repetition penalty for text generation. Default: ``1.0``.
     no_repeat_ngram_size : int
@@ -58,12 +57,8 @@ class Qwen2_5OmniDOA(DecoderOnlyAttention):
     AUDIO_START_TOKEN_ID = 151647
     AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
-        "You are a speech translation system. "
-        "Translate the audio input into the target language. "
-        "Output only the translation. "
-        "Do not ask questions, do not add commentary, do not simulate a conversation, "
-        "do not write 'Human:', 'Assistant:', or any dialogue markers, including newlines. "
-        "If the audio is unclear or incomplete, output only what you can translate and stop."
+        "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
+        "perceiving auditory and visual inputs, as well as generating text and speech."
     )
 
     def __init__(self, config: SimpleNamespace):
@@ -72,7 +67,7 @@ def __init__(self, config: SimpleNamespace):
         text_history_cls = class_load(self.text_history_config.type)
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
-        self.use_video =  getattr(self.config, "use_video", False)
+        self.use_video = getattr(self.config, "use_video", False)
         self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.05)
         self.temperature = getattr(self.config, "temperature", 1.0)
         self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 5)
@@ -82,21 +77,25 @@ def load_model(cls, config: SimpleNamespace) -> None:
         model_name = getattr(
             config,
             "hf_model_name",
-            getattr(config, "model_path", "Qwen/Qwen2.5-Omni-7B"),
+            getattr(config, "model_path", "Qwen/Qwen3-Omni-30B-A3B-Instruct"),
         )
-        attn_impl = getattr(config, "attn_implementation", "eager") #"flash_attention_2")
+        attn_impl = getattr(config, "attn_implementation", "eager")
 
-        cls.model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
+        cls.model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(
             model_name,
             torch_dtype="auto",
             device_map="auto",
             attn_implementation=attn_impl,
         )
-        cls.processor = Qwen2_5OmniProcessor.from_pretrained(model_name)
+        cls.processor = Qwen3OmniMoeProcessor.from_pretrained(model_name)
         cls.model.eval()
 
     def build_prompt(self) -> str:
-        return f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
+        return (
+            TEMPLATED_SPEECH_PROMPT
+            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
+            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
+        )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         prompt_text = self.build_prompt()
@@ -144,18 +143,18 @@ def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         end_positions = (input_ids[0] == self.AUDIO_END_TOKEN_ID).nonzero(as_tuple=True)[0]
         if start_positions.numel() == 0 or end_positions.numel() == 0:
             raise ValueError(
-                "Qwen2.5-Omni audio tokens were not found in the prompt. Checked "
+                "Qwen3-Omni audio tokens were not found in the prompt. Checked "
                 "`audio_token_index`, `<|audio_bos|>`, and `<|audio_eos|>`."
             )
 
         start_pos = start_positions[0]
         end_positions = end_positions[end_positions > start_pos]
         if end_positions.numel() == 0:
-            raise ValueError("Qwen2.5-Omni found `<|audio_bos|>` but not a matching `<|audio_eos|>`.")
+            raise ValueError("Qwen3-Omni found `<|audio_bos|>` but not a matching `<|audio_eos|>`.")
 
         end_pos = end_positions[0]
         if end_pos <= start_pos + 1:
-            raise ValueError("Qwen2.5-Omni found empty audio span between `<|audio_bos|>` and `<|audio_eos|>`.")
+            raise ValueError("Qwen3-Omni found empty audio span between `<|audio_bos|>` and `<|audio_eos|>`.")
 
         return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
 
@@ -176,7 +175,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
-            #thinker_eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
+            #thinker_eos_token_id=[151643, 151645],
             temperature=self.temperature,
         )
         if isinstance(output, tuple):
@@ -210,4 +209,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         return new_tokens, cross_attn
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
+        return "".join(tokens)
\ No newline at end of file

From 0aefbf9beec682de6078e7614ae8ba33ca5b21b0 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 11:43:34 +0200
Subject: [PATCH 099/157] Try new Qwen3Omni

---
 ...7b_doa_punctuation.yaml => qwen3omni_doa_punctuation.yaml} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename config/{qwen2.5omni_7b_doa_punctuation.yaml => qwen3omni_doa_punctuation.yaml} (81%)

diff --git a/config/qwen2.5omni_7b_doa_punctuation.yaml b/config/qwen3omni_doa_punctuation.yaml
similarity index 81%
rename from config/qwen2.5omni_7b_doa_punctuation.yaml
rename to config/qwen3omni_doa_punctuation.yaml
index 77181a3..73fd41d 100644
--- a/config/qwen2.5omni_7b_doa_punctuation.yaml
+++ b/config/qwen3omni_doa_punctuation.yaml
@@ -1,4 +1,4 @@
-type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
+type: "simulstream.server.speech_processors.qwenomni_doa.Qwen3OmniDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
 audio_history_max_duration: 90  # Maximum length for the audio buffer, in seconds
@@ -9,7 +9,7 @@ attn_head: null  # Optional specific head; null means average over heads
 average_attn_over_layers: True
 cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
-hf_model_name: "Qwen/Qwen2.5-Omni-7B"
+hf_model_name: "Qwen/Qwen3-Omni-30B-A3B-Instruct"
 use_video: False
 word_level_postprocess: True  # Disable if character-level language
 max_new_tokens: 32
\ No newline at end of file

From f1772d8a0367ac836cc885992c579210e5134173 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 12:05:50 +0200
Subject: [PATCH 100/157] Try new Qwen3Omni

---
 .../server/speech_processors/qwenomni_doa.py  | 37 ++++---------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 4560ba8..69ba1e9 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -46,16 +46,14 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
     hf_model_name : str
         Default: ``"Qwen/Qwen3-Omni-30B-A3B-Instruct"``.
     repetition_penalty : float
-        Repetition penalty for text generation. Default: ``1.0``.
+        Repetition penalty for text generation. Default: ``1.05``.
     no_repeat_ngram_size : int
-        N-gram blocking size for text generation. Default: ``0``.
+        N-gram blocking size for text generation. Default: ``5``.
     """
 
     BOW_PREFIX = " "
     AUDIO_TOKEN_STRIDE = 640
     AUDIO_TOKEN_INDEX = 151646
-    AUDIO_START_TOKEN_ID = 151647
-    AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
         "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
         "perceiving auditory and visual inputs, as well as generating text and speech."
@@ -86,6 +84,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
             torch_dtype="auto",
             device_map="auto",
             attn_implementation=attn_impl,
+            enable_audio_output=False,
         )
         cls.processor = Qwen3OmniMoeProcessor.from_pretrained(model_name)
         cls.model.eval()
@@ -99,6 +98,7 @@ def build_prompt(self) -> str:
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         prompt_text = self.build_prompt()
+        prefix = self.build_raw_text_prefix()
 
         conversation = [
             {
@@ -119,11 +119,10 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             add_generation_prompt=True,
             tokenize=False,
         )
-        prefix = self.build_raw_text_prefix()
 
         audios, images, videos = process_mm_info(conversation, use_audio_in_video=True)
 
-        return self.processor(
+        inputs = self.processor(
             text=f"{prompt}{prefix}",
             audio=audios,
             images=images,
@@ -132,31 +131,11 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             return_tensors="pt",
             padding=True,
             use_audio_in_video=True,
-        ).to(self.device)
+        )
+        return inputs.to(self.device).to(self.model.dtype)
 
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
-        audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
-        if audio_positions.numel() > 0:
-            return audio_positions
-
-        start_positions = (input_ids[0] == self.AUDIO_START_TOKEN_ID).nonzero(as_tuple=True)[0]
-        end_positions = (input_ids[0] == self.AUDIO_END_TOKEN_ID).nonzero(as_tuple=True)[0]
-        if start_positions.numel() == 0 or end_positions.numel() == 0:
-            raise ValueError(
-                "Qwen3-Omni audio tokens were not found in the prompt. Checked "
-                "`audio_token_index`, `<|audio_bos|>`, and `<|audio_eos|>`."
-            )
-
-        start_pos = start_positions[0]
-        end_positions = end_positions[end_positions > start_pos]
-        if end_positions.numel() == 0:
-            raise ValueError("Qwen3-Omni found `<|audio_bos|>` but not a matching `<|audio_eos|>`.")
-
-        end_pos = end_positions[0]
-        if end_pos <= start_pos + 1:
-            raise ValueError("Qwen3-Omni found empty audio span between `<|audio_bos|>` and `<|audio_eos|>`.")
-
-        return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
+        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
 
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         input_ids = inputs["input_ids"]

From 1658514456da164a4ccb98b541a19a6e23b75a66 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 12:30:50 +0200
Subject: [PATCH 101/157] Try new Qwen3Omni

---
 simulstream/server/speech_processors/qwenomni_doa.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 69ba1e9..e37f917 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -168,19 +168,19 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
+
+        empty_attn = torch.zeros(0, audio_len, device=self.device)
         if prefix_len > 0:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
         else:
-            prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
+            prefix_rows = empty_attn
 
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]
         ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)

From d5d1807f6e1e493582e397d3eb814d5f970c0c43 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 12:52:28 +0200
Subject: [PATCH 102/157] Try new Qwen3Omni

---
 .../server/speech_processors/qwenomni_doa.py  | 37 +++++++++++++++----
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index e37f917..6a26833 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -54,6 +54,8 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
     BOW_PREFIX = " "
     AUDIO_TOKEN_STRIDE = 640
     AUDIO_TOKEN_INDEX = 151646
+    AUDIO_START_TOKEN_ID = 151647
+    AUDIO_END_TOKEN_ID = 151648
     SYSTEM_PROMPT = (
         "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
         "perceiving auditory and visual inputs, as well as generating text and speech."
@@ -135,7 +137,28 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         return inputs.to(self.device).to(self.model.dtype)
 
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
-        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+        audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+        if audio_positions.numel() > 0:
+            return audio_positions
+
+        start_positions = (input_ids[0] == self.AUDIO_START_TOKEN_ID).nonzero(as_tuple=True)[0]
+        end_positions = (input_ids[0] == self.AUDIO_END_TOKEN_ID).nonzero(as_tuple=True)[0]
+        if start_positions.numel() == 0 or end_positions.numel() == 0:
+            raise ValueError(
+                "Qwen3-Omni audio tokens were not found in the prompt. Checked "
+                "`audio_token_index`, `<|audio_bos|>`, and `<|audio_eos|>`."
+            )
+
+        start_pos = start_positions[0]
+        end_positions = end_positions[end_positions > start_pos]
+        if end_positions.numel() == 0:
+            raise ValueError("Qwen3-Omni found `<|audio_bos|>` but not a matching `<|audio_eos|>`.")
+
+        end_pos = end_positions[0]
+        if end_pos <= start_pos + 1:
+            raise ValueError("Qwen3-Omni found empty audio span between `<|audio_bos|>` and `<|audio_eos|>`.")
+
+        return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
 
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         input_ids = inputs["input_ids"]
@@ -168,19 +191,19 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
-
-        empty_attn = torch.zeros(0, audio_len, device=self.device)
         if prefix_len > 0:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
         else:
-            prefix_rows = empty_attn
+            prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
 
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]
         ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
+            torch.zeros(0, max(audio_len, 1), device=self.device)
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
@@ -188,4 +211,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         return new_tokens, cross_attn
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
\ No newline at end of file
+        return "".join(tokens)

From 13bd2e3ef4d1e667b7784920d29e7050e5faf0c9 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 12:55:57 +0200
Subject: [PATCH 103/157] Try new Qwen3Omni

---
 .../server/speech_processors/qwenomni_doa.py  | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 6a26833..5967819 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -53,6 +53,9 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
 
     BOW_PREFIX = " "
     AUDIO_TOKEN_STRIDE = 640
+    AUDIO_TOKEN = "<|AUDIO|>"
+    AUDIO_START_TOKEN = "<|audio_bos|>"
+    AUDIO_END_TOKEN = "<|audio_eos|>"
     AUDIO_TOKEN_INDEX = 151646
     AUDIO_START_TOKEN_ID = 151647
     AUDIO_END_TOKEN_ID = 151648
@@ -72,6 +75,19 @@ def __init__(self, config: SimpleNamespace):
         self.temperature = getattr(self.config, "temperature", 1.0)
         self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 5)
 
+    @classmethod
+    def _resolve_special_token_id(cls, token: str, fallback_id: int) -> int:
+        tokenizer = cls.processor.tokenizer
+        token_id = tokenizer.get_vocab().get(token)
+        if token_id is not None:
+            return token_id
+
+        converted_id = tokenizer.convert_tokens_to_ids(token)
+        unk_token_id = getattr(tokenizer, "unk_token_id", None)
+        if converted_id is not None and converted_id != unk_token_id:
+            return converted_id
+        return fallback_id
+
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
         model_name = getattr(
@@ -89,6 +105,18 @@ def load_model(cls, config: SimpleNamespace) -> None:
             enable_audio_output=False,
         )
         cls.processor = Qwen3OmniMoeProcessor.from_pretrained(model_name)
+        cls.AUDIO_TOKEN_INDEX = cls._resolve_special_token_id(
+            cls.AUDIO_TOKEN,
+            cls.AUDIO_TOKEN_INDEX,
+        )
+        cls.AUDIO_START_TOKEN_ID = cls._resolve_special_token_id(
+            cls.AUDIO_START_TOKEN,
+            cls.AUDIO_START_TOKEN_ID,
+        )
+        cls.AUDIO_END_TOKEN_ID = cls._resolve_special_token_id(
+            cls.AUDIO_END_TOKEN,
+            cls.AUDIO_END_TOKEN_ID,
+        )
         cls.model.eval()
 
     def build_prompt(self) -> str:

From c5981e674b8ed4847a847b09bbeb147143bb020a Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 13:08:16 +0200
Subject: [PATCH 104/157] Try new Qwen3Omni

---
 .../server/speech_processors/qwenomni_doa.py  | 69 ++++---------------
 1 file changed, 12 insertions(+), 57 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 5967819..6d14074 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -53,12 +53,9 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
 
     BOW_PREFIX = " "
     AUDIO_TOKEN_STRIDE = 640
-    AUDIO_TOKEN = "<|AUDIO|>"
-    AUDIO_START_TOKEN = "<|audio_bos|>"
-    AUDIO_END_TOKEN = "<|audio_eos|>"
-    AUDIO_TOKEN_INDEX = 151646
-    AUDIO_START_TOKEN_ID = 151647
-    AUDIO_END_TOKEN_ID = 151648
+    AUDIO_TOKEN_INDEX = 151675   # <|audio_pad|>
+    AUDIO_START_TOKEN_ID = 151669  # <|audio_start|>
+    AUDIO_END_TOKEN_ID = 151670    # <|audio_end|>
     SYSTEM_PROMPT = (
         "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
         "perceiving auditory and visual inputs, as well as generating text and speech."
@@ -75,19 +72,6 @@ def __init__(self, config: SimpleNamespace):
         self.temperature = getattr(self.config, "temperature", 1.0)
         self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 5)
 
-    @classmethod
-    def _resolve_special_token_id(cls, token: str, fallback_id: int) -> int:
-        tokenizer = cls.processor.tokenizer
-        token_id = tokenizer.get_vocab().get(token)
-        if token_id is not None:
-            return token_id
-
-        converted_id = tokenizer.convert_tokens_to_ids(token)
-        unk_token_id = getattr(tokenizer, "unk_token_id", None)
-        if converted_id is not None and converted_id != unk_token_id:
-            return converted_id
-        return fallback_id
-
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
         model_name = getattr(
@@ -105,18 +89,6 @@ def load_model(cls, config: SimpleNamespace) -> None:
             enable_audio_output=False,
         )
         cls.processor = Qwen3OmniMoeProcessor.from_pretrained(model_name)
-        cls.AUDIO_TOKEN_INDEX = cls._resolve_special_token_id(
-            cls.AUDIO_TOKEN,
-            cls.AUDIO_TOKEN_INDEX,
-        )
-        cls.AUDIO_START_TOKEN_ID = cls._resolve_special_token_id(
-            cls.AUDIO_START_TOKEN,
-            cls.AUDIO_START_TOKEN_ID,
-        )
-        cls.AUDIO_END_TOKEN_ID = cls._resolve_special_token_id(
-            cls.AUDIO_END_TOKEN,
-            cls.AUDIO_END_TOKEN_ID,
-        )
         cls.model.eval()
 
     def build_prompt(self) -> str:
@@ -169,23 +141,9 @@ def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         if audio_positions.numel() > 0:
             return audio_positions
 
-        start_positions = (input_ids[0] == self.AUDIO_START_TOKEN_ID).nonzero(as_tuple=True)[0]
-        end_positions = (input_ids[0] == self.AUDIO_END_TOKEN_ID).nonzero(as_tuple=True)[0]
-        if start_positions.numel() == 0 or end_positions.numel() == 0:
-            raise ValueError(
-                "Qwen3-Omni audio tokens were not found in the prompt. Checked "
-                "`audio_token_index`, `<|audio_bos|>`, and `<|audio_eos|>`."
-            )
-
-        start_pos = start_positions[0]
-        end_positions = end_positions[end_positions > start_pos]
-        if end_positions.numel() == 0:
-            raise ValueError("Qwen3-Omni found `<|audio_bos|>` but not a matching `<|audio_eos|>`.")
-
-        end_pos = end_positions[0]
-        if end_pos <= start_pos + 1:
-            raise ValueError("Qwen3-Omni found empty audio span between `<|audio_bos|>` and `<|audio_eos|>`.")
-
+        start_pos = (input_ids[0] == self.AUDIO_START_TOKEN_ID).nonzero(as_tuple=True)[0][0]
+        end_pos = (input_ids[0] == self.AUDIO_END_TOKEN_ID).nonzero(as_tuple=True)[0]
+        end_pos = end_pos[end_pos > start_pos][0]
         return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
 
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
@@ -219,19 +177,16 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
-        if prefix_len > 0:
-            prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
-        else:
-            prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
+        empty_attn = torch.zeros(0, audio_len, device=self.device)
 
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
+        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
+            if prefix_len > 0 else empty_attn
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]
         ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
         cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
@@ -239,4 +194,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         return new_tokens, cross_attn
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
+        return "".join(tokens)
\ No newline at end of file

From c4297e23d45856a0bd7e98f3b0b204cbd2c0bc6a Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 18:51:03 +0200
Subject: [PATCH 105/157] Trim max input length of Qwen3Omni

---
 config/qwen3omni_doa_punctuation.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/qwen3omni_doa_punctuation.yaml b/config/qwen3omni_doa_punctuation.yaml
index 73fd41d..443f4bc 100644
--- a/config/qwen3omni_doa_punctuation.yaml
+++ b/config/qwen3omni_doa_punctuation.yaml
@@ -1,7 +1,7 @@
 type: "simulstream.server.speech_processors.qwenomni_doa.Qwen3OmniDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 90  # Maximum length for the audio buffer, in seconds
+audio_history_max_duration: 60  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__

From c7985373b5c955b1fe23c33992461287c6db45af Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Mon, 18 May 2026 22:18:53 +0200
Subject: [PATCH 106/157] Add Gemma4

---
 config/gemma4_doa_punctuation.yaml            |  14 ++
 .../server/speech_processors/gemma4_doa.py    | 177 ++++++++++++++++++
 2 files changed, 191 insertions(+)
 create mode 100644 config/gemma4_doa_punctuation.yaml
 create mode 100644 simulstream/server/speech_processors/gemma4_doa.py

diff --git a/config/gemma4_doa_punctuation.yaml b/config/gemma4_doa_punctuation.yaml
new file mode 100644
index 0000000..d49029d
--- /dev/null
+++ b/config/gemma4_doa_punctuation.yaml
@@ -0,0 +1,14 @@
+type: "simulstream.server.speech_processors.gemma4_doa.Gemma4DOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 60  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+attn_head: null  # Optional specific head; null means average over heads
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
+detokenizer_type: "hf"
+hf_model_name: "google/gemma-4-E2B-it"
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/gemma4_doa.py b/simulstream/server/speech_processors/gemma4_doa.py
new file mode 100644
index 0000000..3f9a0ac
--- /dev/null
+++ b/simulstream/server/speech_processors/gemma4_doa.py
@@ -0,0 +1,177 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import logging
+from types import SimpleNamespace
+from typing import List, Tuple
+
+import numpy as np
+import torch
+
+from transformers import AutoModelForMultimodalLM, AutoProcessor
+
+from simulstream.server.speech_processors import class_load
+from simulstream.server.speech_processors.base_doa import (
+    DecoderOnlyAttention,
+    LANG_MAPPER,
+    TEMPLATED_SPEECH_PROMPT,
+)
+
+from transformers import set_seed
+torch.manual_seed(42)
+set_seed(42)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Gemma4DOA(DecoderOnlyAttention):
+    """
+    Decoder-Only Attention agent for ``google/gemma-4-E2B-it`` and
+    ``google/gemma-4-E4B-it`` (the only Gemma 4 variants with audio support).
+
+    Extra config fields
+    -------------------
+    hf_model_name : str
+        Default: ``"google/gemma-4-E2B-it"``.
+    repetition_penalty : float
+        Repetition penalty for text generation. Default: ``1.0``.
+    no_repeat_ngram_size : int
+        N-gram blocking size. Default: ``0``.
+    """
+
+    BOW_PREFIX = " "
+    AUDIO_TOKEN_INDEX = 258881  # <|audio|>
+    AUDIO_TOKEN_STRIDE = 640    # 16000 / 25 tokens per second
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.bow_prefix = self.BOW_PREFIX
+        text_history_cls = class_load(self.text_history_config.type)
+        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
+        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
+        self.temperature = getattr(self.config, "temperature", 1.0)
+        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
+
+    @classmethod
+    def load_model(cls, config: SimpleNamespace) -> None:
+        model_name = getattr(
+            config,
+            "hf_model_name",
+            getattr(config, "model_path", "google/gemma-4-E2B-it"),
+        )
+        attn_impl = getattr(config, "attn_implementation", "eager")
+
+        cls.processor = AutoProcessor.from_pretrained(model_name)
+        cls.model = AutoModelForMultimodalLM.from_pretrained(
+            model_name,
+            torch_dtype="auto",
+            device_map="auto",
+            attn_implementation=attn_impl,
+        )
+        cls.model.eval()
+
+    def build_prompt(self) -> str:
+        return (
+            TEMPLATED_SPEECH_PROMPT
+            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
+            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
+        )
+
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        prefix = self.build_raw_text_prefix()
+
+        conversation = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "audio", "audio": waveform},
+                    {"type": "text", "text": self.build_prompt()},
+                ],
+            },
+        ]
+
+        inputs = self.processor.apply_chat_template(
+            conversation,
+            tokenize=True,
+            return_dict=True,
+            return_tensors="pt",
+            add_generation_prompt=True,
+            enable_thinking=False,
+        )
+
+        if prefix:
+            prefix_ids = self.processor.tokenizer(
+                prefix,
+                return_tensors="pt",
+                add_special_tokens=False,
+            ).input_ids
+            inputs["input_ids"] = torch.cat([inputs["input_ids"], prefix_ids], dim=1)
+            inputs["attention_mask"] = torch.cat(
+                [inputs["attention_mask"], torch.ones_like(prefix_ids)], dim=1
+            )
+
+        return {
+            k: v.to(self.device) if isinstance(v, torch.Tensor) else v
+            for k, v in inputs.items()
+        }
+
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+
+    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+        input_ids = inputs["input_ids"]
+        input_len = input_ids.shape[1]
+
+        audio_positions = self._find_audio_positions(input_ids)
+        audio_len = audio_positions.shape[0]
+
+        output = self.model.generate(
+            **inputs,
+            max_new_tokens=self.max_new_tokens,
+            repetition_penalty=self.repetition_penalty,
+            no_repeat_ngram_size=self.no_repeat_ngram_size,
+            output_attentions=True,
+            return_dict_in_generate=True,
+            do_sample=False,
+            temperature=self.temperature,
+        )
+
+        new_ids = output.sequences[:, input_len:]
+        new_tokens = [
+            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in new_ids[0]
+        ]
+
+        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
+        prefix_len = len(self.text_history) if self.text_history else 0
+        empty_attn = torch.zeros(0, audio_len, device=self.device)
+
+        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
+            if prefix_len > 0 else empty_attn
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
+        new_rows = [
+            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:]
+        ]
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
+        cross_attn = self.normalize_attn(cross_attn)
+        return new_tokens, cross_attn
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens)
\ No newline at end of file

From 52c8b5981d800a89637644baa569257d18e9350d Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 20 May 2026 10:59:29 +0200
Subject: [PATCH 107/157] Add Gemma4 global layers for the average

---
 simulstream/server/speech_processors/gemma4_doa.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/simulstream/server/speech_processors/gemma4_doa.py b/simulstream/server/speech_processors/gemma4_doa.py
index 3f9a0ac..b834ec0 100644
--- a/simulstream/server/speech_processors/gemma4_doa.py
+++ b/simulstream/server/speech_processors/gemma4_doa.py
@@ -54,6 +54,7 @@ class Gemma4DOA(DecoderOnlyAttention):
     BOW_PREFIX = " "
     AUDIO_TOKEN_INDEX = 258881  # <|audio|>
     AUDIO_TOKEN_STRIDE = 640    # 16000 / 25 tokens per second
+    GLOBAL_LAYERS = [4, 9, 14, 19, 24, 29, 34] # Indices of global (non-sliding) attention layers
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
@@ -131,6 +132,12 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
 
+    def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
+        return torch.stack(
+            [self._select_attn_from_layer(step_attn[i]) for i in self.GLOBAL_LAYERS],
+            dim=0,
+        ).mean(dim=0)
+
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         input_ids = inputs["input_ids"]
         input_len = input_ids.shape[1]

From c0dcf25fb024210376ce6f052eed48d59137c18d Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 20 May 2026 15:57:25 +0200
Subject: [PATCH 108/157] Add Qwen2Audio

---
 ...n.yaml => qwen2audio_doa_punctuation.yaml} |   4 +-
 simulstream/server/qwenaudio_doa.py           | 156 ++++++++++++++++++
 2 files changed, 158 insertions(+), 2 deletions(-)
 rename config/{qwen2.5omni_3b_doa_punctuation.yaml => qwen2audio_doa_punctuation.yaml} (81%)
 create mode 100644 simulstream/server/qwenaudio_doa.py

diff --git a/config/qwen2.5omni_3b_doa_punctuation.yaml b/config/qwen2audio_doa_punctuation.yaml
similarity index 81%
rename from config/qwen2.5omni_3b_doa_punctuation.yaml
rename to config/qwen2audio_doa_punctuation.yaml
index dec535d..b58ff5a 100644
--- a/config/qwen2.5omni_3b_doa_punctuation.yaml
+++ b/config/qwen2audio_doa_punctuation.yaml
@@ -1,4 +1,4 @@
-type: "simulstream.server.speech_processors.qwen2_5_doa.Qwen2_5OmniDOA"
+type: "simulstream.server.speech_processors.qwenaudio.Qwen2AudioDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
 audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
@@ -9,7 +9,7 @@ attn_head: null  # Optional specific head; null means average over heads
 average_attn_over_layers: True
 cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
-hf_model_name: "Qwen/Qwen2.5-Omni-3B"
+hf_model_name: "Qwen/Qwen2-Audio-7B-Instruct"
 use_video: False
 word_level_postprocess: True  # Disable if character-level language
 max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/server/qwenaudio_doa.py b/simulstream/server/qwenaudio_doa.py
new file mode 100644
index 0000000..2ab00de
--- /dev/null
+++ b/simulstream/server/qwenaudio_doa.py
@@ -0,0 +1,156 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import logging
+from types import SimpleNamespace
+from typing import List, Tuple
+
+import numpy as np
+import torch
+
+from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
+
+from simulstream.server.speech_processors import SAMPLE_RATE, class_load
+from simulstream.server.speech_processors.base_doa import (
+    DecoderOnlyAttention,
+    LANG_MAPPER,
+    TEMPLATED_SPEECH_PROMPT,
+)
+
+from transformers import set_seed
+torch.manual_seed(42)
+set_seed(42)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Qwen2AudioDOA(DecoderOnlyAttention):
+    """
+    Decoder-Only Attention agent for ``Qwen/Qwen2-Audio-7B-Instruct``.
+
+    Architecture: Whisper encoder (subsampling factor 2) → linear projector → Qwen2-7B.
+    Audio is represented as repeated ``<|AUDIO|>`` (id=151646) placeholder tokens
+    in ``input_ids`` after processor expansion.
+
+    Extra config fields
+    -------------------
+    hf_model_name : str
+        Default: ``"Qwen/Qwen2-Audio-7B-Instruct"``.
+    repetition_penalty : float
+        Default: ``1.0``.
+    no_repeat_ngram_size : int
+        Default: ``0``.
+    """
+
+    BOW_PREFIX = " "
+    # Whisper encoder: 50 frames/s, subsampling factor 2 → 25 tokens/s
+    # stride = 16000 / 25 = 640 samples per audio token
+    AUDIO_TOKEN_STRIDE = 640
+    AUDIO_TOKEN_INDEX = 151646  # <|AUDIO|>
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.bow_prefix = self.BOW_PREFIX
+        text_history_cls = class_load(self.text_history_config.type)
+        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
+        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
+        self.temperature = getattr(self.config, "temperature", 1.0)
+        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
+
+    @classmethod
+    def load_model(cls, config: SimpleNamespace) -> None:
+        model_name = getattr(
+            config,
+            "hf_model_name",
+            getattr(config, "model_path", "Qwen/Qwen2-Audio-7B-Instruct"),
+        )
+        attn_impl = getattr(config, "attn_implementation", "eager")
+
+        cls.processor = AutoProcessor.from_pretrained(model_name)
+        cls.model = Qwen2AudioForConditionalGeneration.from_pretrained(
+            model_name,
+            torch_dtype="auto",
+            device_map="auto",
+            attn_implementation=attn_impl,
+        )
+        cls.model.eval()
+
+    def build_prompt(self) -> str:
+        return (
+            TEMPLATED_SPEECH_PROMPT
+            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
+            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
+        )
+
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        prefix = self.build_raw_text_prefix()
+        prompt = f"<|audio_bos|><|AUDIO|><|audio_eos|>{self.build_prompt()}"
+
+        inputs = self.processor(
+            text=f"{prompt}{prefix}",
+            audios=waveform,
+            sampling_rate=SAMPLE_RATE,
+            return_tensors="pt",
+        )
+        return inputs.to(self.device)
+
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+
+    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+        input_ids = inputs["input_ids"]
+        input_len = input_ids.shape[1]
+
+        audio_positions = self._find_audio_positions(input_ids)
+        audio_len = audio_positions.shape[0]
+
+        output = self.model.generate(
+            **inputs,
+            max_new_tokens=self.max_new_tokens,
+            repetition_penalty=self.repetition_penalty,
+            no_repeat_ngram_size=self.no_repeat_ngram_size,
+            output_attentions=True,
+            return_dict_in_generate=True,
+            do_sample=False,
+            temperature=self.temperature,
+        )
+
+        new_ids = output.sequences[:, input_len:]
+        new_tokens = [
+            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in new_ids[0]
+        ]
+
+        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
+        prefix_len = len(self.text_history) if self.text_history else 0
+        empty_attn = torch.zeros(0, audio_len, device=self.device)
+
+        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
+            if prefix_len > 0 else empty_attn
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
+        new_rows = [
+            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:]
+        ]
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
+        cross_attn = self.normalize_attn(cross_attn)
+        return new_tokens, cross_attn
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens)
\ No newline at end of file

From 44d8e1efa8a0fa3a19247084ae56e370d4cfb991 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 20 May 2026 16:25:03 +0200
Subject: [PATCH 109/157] Fix wrong path

---
 config/qwen2audio_doa_punctuation.yaml                      | 2 +-
 simulstream/server/{ => speech_processors}/qwenaudio_doa.py | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename simulstream/server/{ => speech_processors}/qwenaudio_doa.py (100%)

diff --git a/config/qwen2audio_doa_punctuation.yaml b/config/qwen2audio_doa_punctuation.yaml
index b58ff5a..cadad1b 100644
--- a/config/qwen2audio_doa_punctuation.yaml
+++ b/config/qwen2audio_doa_punctuation.yaml
@@ -1,4 +1,4 @@
-type: "simulstream.server.speech_processors.qwenaudio.Qwen2AudioDOA"
+type: "simulstream.server.speech_processors.qwenaudio_doa.Qwen2AudioDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
 audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
diff --git a/simulstream/server/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
similarity index 100%
rename from simulstream/server/qwenaudio_doa.py
rename to simulstream/server/speech_processors/qwenaudio_doa.py

From 80b9bd339959687a3259dfd1c322c23717a4ea7b Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 20 May 2026 17:29:54 +0200
Subject: [PATCH 110/157] Fix Qwen2Audio inference

---
 .../server/speech_processors/qwenaudio_doa.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
index 2ab00de..e0ccd89 100644
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ b/simulstream/server/speech_processors/qwenaudio_doa.py
@@ -41,8 +41,9 @@ class Qwen2AudioDOA(DecoderOnlyAttention):
     Decoder-Only Attention agent for ``Qwen/Qwen2-Audio-7B-Instruct``.
 
     Architecture: Whisper encoder (subsampling factor 2) → linear projector → Qwen2-7B.
-    Audio is represented as repeated ``<|AUDIO|>`` (id=151646) placeholder tokens
-    in ``input_ids`` after processor expansion.
+    Audio is serialized through the official Qwen2-Audio chat template and then
+    expanded by the processor into repeated ``<|AUDIO|>`` placeholder tokens in
+    ``input_ids``.
 
     Extra config fields
     -------------------
@@ -97,13 +98,19 @@ def build_prompt(self) -> str:
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         prefix = self.build_raw_text_prefix()
-        prompt = f"<|audio_bos|><|AUDIO|><|audio_eos|>{self.build_prompt()}"
+        prompt = (
+            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n"
+            "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
+            f"{self.build_prompt()}<|im_end|>\n"
+            f"<|im_start|>assistant\n{prefix}"
+        )
 
         inputs = self.processor(
-            text=f"{prompt}{prefix}",
-            audios=waveform,
+            text=prompt,
+            audios=[waveform],
             sampling_rate=SAMPLE_RATE,
             return_tensors="pt",
+            padding=True,
         )
         return inputs.to(self.device)
 
@@ -153,4 +160,4 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         return new_tokens, cross_attn
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
\ No newline at end of file
+        return "".join(tokens)

From 3ba31fe357d219d525780324bd13123a546d76a1 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 20 May 2026 18:06:45 +0200
Subject: [PATCH 111/157] Fix Qwen2Audio inference

---
 .../server/speech_processors/qwenaudio_doa.py | 24 +++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
index e0ccd89..72a5d23 100644
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ b/simulstream/server/speech_processors/qwenaudio_doa.py
@@ -114,16 +114,28 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         )
         return inputs.to(self.device)
 
-    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
-        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+    def _find_audio_positions(self, input_ids: torch.Tensor, prefill_len: int) -> torch.Tensor:
+        raw_audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+        if raw_audio_positions.numel() == 0:
+            raise ValueError("Qwen2-Audio audio placeholder token was not found in input_ids.")
+
+        if raw_audio_positions.numel() > 1 or prefill_len == input_ids.shape[1]:
+            return raw_audio_positions
+
+        expanded_audio_len = prefill_len - input_ids.shape[1] + 1
+        if expanded_audio_len <= 0:
+            raise ValueError(
+                "Qwen2-Audio audio expansion length is invalid. "
+                f"prefill_len={prefill_len}, input_len={input_ids.shape[1]}."
+            )
+
+        start = raw_audio_positions[0].item()
+        return torch.arange(start, start + expanded_audio_len, device=input_ids.device)
 
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         input_ids = inputs["input_ids"]
         input_len = input_ids.shape[1]
 
-        audio_positions = self._find_audio_positions(input_ids)
-        audio_len = audio_positions.shape[0]
-
         output = self.model.generate(
             **inputs,
             max_new_tokens=self.max_new_tokens,
@@ -142,6 +154,8 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         ]
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
+        audio_positions = self._find_audio_positions(input_ids, prefill_attn.shape[0])
+        audio_len = audio_positions.shape[0]
         prefix_len = len(self.text_history) if self.text_history else 0
         empty_attn = torch.zeros(0, audio_len, device=self.device)
 

From 2e2b62d0c7b3475b449e0ba47823b2a46cfd1148 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 10:14:23 +0200
Subject: [PATCH 112/157] Fix Qwen2Audio inference

---
 .../server/speech_processors/qwenaudio_doa.py      | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
index 72a5d23..f61b924 100644
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ b/simulstream/server/speech_processors/qwenaudio_doa.py
@@ -114,11 +114,21 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         )
         return inputs.to(self.device)
 
-    def _find_audio_positions(self, input_ids: torch.Tensor, prefill_len: int) -> torch.Tensor:
+    def _find_audio_positions(self, inputs: dict, prefill_len: int) -> torch.Tensor:
+        input_ids = inputs["input_ids"]
         raw_audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
         if raw_audio_positions.numel() == 0:
             raise ValueError("Qwen2-Audio audio placeholder token was not found in input_ids.")
 
+        feature_attention_mask = inputs.get("feature_attention_mask")
+        if feature_attention_mask is not None:
+            audio_feat_lengths, audio_output_lengths = self.model.audio_tower._get_feat_extract_output_lengths(
+                feature_attention_mask.sum(-1)
+            )
+            valid_audio_len = int(audio_output_lengths[0].item())
+            if 0 < valid_audio_len <= raw_audio_positions.numel():
+                return raw_audio_positions[:valid_audio_len]
+
         if raw_audio_positions.numel() > 1 or prefill_len == input_ids.shape[1]:
             return raw_audio_positions
 
@@ -154,7 +164,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         ]
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        audio_positions = self._find_audio_positions(input_ids, prefill_attn.shape[0])
+        audio_positions = self._find_audio_positions(inputs, prefill_attn.shape[0])
         audio_len = audio_positions.shape[0]
         prefix_len = len(self.text_history) if self.text_history else 0
         empty_attn = torch.zeros(0, audio_len, device=self.device)

From 96128c35f808d7e699ae34a202c81d147b833c1a Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 10:14:35 +0200
Subject: [PATCH 113/157] seed

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 72363a3..0712d20 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -24,8 +24,8 @@
 from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, LANG_MAPPER
 
 from transformers import set_seed
-torch.manual_seed(42)
-set_seed(42)
+torch.manual_seed(41)
+set_seed(41)
 
 
 class Phi4MultimodalDOA(DecoderOnlyAttention):

From 9a2755f18b5918705263dede3c2b98d1c55b3eb5 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 12:03:47 +0200
Subject: [PATCH 114/157] Revert "Fix Qwen2Audio inference"

This reverts commit 2e2b62d0c7b3475b449e0ba47823b2a46cfd1148.
---
 .../server/speech_processors/qwenaudio_doa.py      | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
index f61b924..72a5d23 100644
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ b/simulstream/server/speech_processors/qwenaudio_doa.py
@@ -114,21 +114,11 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         )
         return inputs.to(self.device)
 
-    def _find_audio_positions(self, inputs: dict, prefill_len: int) -> torch.Tensor:
-        input_ids = inputs["input_ids"]
+    def _find_audio_positions(self, input_ids: torch.Tensor, prefill_len: int) -> torch.Tensor:
         raw_audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
         if raw_audio_positions.numel() == 0:
             raise ValueError("Qwen2-Audio audio placeholder token was not found in input_ids.")
 
-        feature_attention_mask = inputs.get("feature_attention_mask")
-        if feature_attention_mask is not None:
-            audio_feat_lengths, audio_output_lengths = self.model.audio_tower._get_feat_extract_output_lengths(
-                feature_attention_mask.sum(-1)
-            )
-            valid_audio_len = int(audio_output_lengths[0].item())
-            if 0 < valid_audio_len <= raw_audio_positions.numel():
-                return raw_audio_positions[:valid_audio_len]
-
         if raw_audio_positions.numel() > 1 or prefill_len == input_ids.shape[1]:
             return raw_audio_positions
 
@@ -164,7 +154,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         ]
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        audio_positions = self._find_audio_positions(inputs, prefill_attn.shape[0])
+        audio_positions = self._find_audio_positions(input_ids, prefill_attn.shape[0])
         audio_len = audio_positions.shape[0]
         prefix_len = len(self.text_history) if self.text_history else 0
         empty_attn = torch.zeros(0, audio_len, device=self.device)

From eefe1334cbdb000239b9d8ae505e3d20986664d2 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 12:03:53 +0200
Subject: [PATCH 115/157] Revert "Fix Qwen2Audio inference"

This reverts commit 3ba31fe357d219d525780324bd13123a546d76a1.
---
 .../server/speech_processors/qwenaudio_doa.py | 24 ++++---------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
index 72a5d23..e0ccd89 100644
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ b/simulstream/server/speech_processors/qwenaudio_doa.py
@@ -114,28 +114,16 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         )
         return inputs.to(self.device)
 
-    def _find_audio_positions(self, input_ids: torch.Tensor, prefill_len: int) -> torch.Tensor:
-        raw_audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
-        if raw_audio_positions.numel() == 0:
-            raise ValueError("Qwen2-Audio audio placeholder token was not found in input_ids.")
-
-        if raw_audio_positions.numel() > 1 or prefill_len == input_ids.shape[1]:
-            return raw_audio_positions
-
-        expanded_audio_len = prefill_len - input_ids.shape[1] + 1
-        if expanded_audio_len <= 0:
-            raise ValueError(
-                "Qwen2-Audio audio expansion length is invalid. "
-                f"prefill_len={prefill_len}, input_len={input_ids.shape[1]}."
-            )
-
-        start = raw_audio_positions[0].item()
-        return torch.arange(start, start + expanded_audio_len, device=input_ids.device)
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
 
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         input_ids = inputs["input_ids"]
         input_len = input_ids.shape[1]
 
+        audio_positions = self._find_audio_positions(input_ids)
+        audio_len = audio_positions.shape[0]
+
         output = self.model.generate(
             **inputs,
             max_new_tokens=self.max_new_tokens,
@@ -154,8 +142,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         ]
 
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        audio_positions = self._find_audio_positions(input_ids, prefill_attn.shape[0])
-        audio_len = audio_positions.shape[0]
         prefix_len = len(self.text_history) if self.text_history else 0
         empty_attn = torch.zeros(0, audio_len, device=self.device)
 

From f71fed990407c9d4884b05c4a52b7432fcd10464 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 13:54:11 +0200
Subject: [PATCH 116/157] Fix Qwen2Audio

---
 .../server/speech_processors/qwenaudio_doa.py | 27 +++++++++++++------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
index e0ccd89..3936856 100644
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ b/simulstream/server/speech_processors/qwenaudio_doa.py
@@ -98,17 +98,27 @@ def build_prompt(self) -> str:
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         prefix = self.build_raw_text_prefix()
-        prompt = (
-            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n"
-            "Audio 1: <|audio_bos|><|AUDIO|><|audio_eos|>\n"
-            f"{self.build_prompt()}<|im_end|>\n"
-            f"<|im_start|>assistant\n{prefix}"
+
+        conversation = [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {
+                "role": "user",
+                "content": [
+                    {"type": "audio", "audio_url": "placeholder"},
+                    {"type": "text", "text": self.build_prompt()},
+                ],
+            },
+        ]
+
+        text = self.processor.apply_chat_template(
+            conversation,
+            add_generation_prompt=True,
+            tokenize=False,
         )
 
         inputs = self.processor(
-            text=prompt,
-            audios=[waveform],
-            sampling_rate=SAMPLE_RATE,
+            text=f"{text}{prefix}",
+            audio=[waveform],
             return_tensors="pt",
             padding=True,
         )
@@ -133,6 +143,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             return_dict_in_generate=True,
             do_sample=False,
             temperature=self.temperature,
+            eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
         )
 
         new_ids = output.sequences[:, input_len:]

From b5eabf5afde6d6f3c4dc0320330178deb52994ec Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 14:49:58 +0200
Subject: [PATCH 117/157] Remove Gemma4 as it performs very bad

---
 config/gemma4_doa_punctuation.yaml            |  14 --
 .../server/speech_processors/gemma4_doa.py    | 184 ------------------
 2 files changed, 198 deletions(-)
 delete mode 100644 config/gemma4_doa_punctuation.yaml
 delete mode 100644 simulstream/server/speech_processors/gemma4_doa.py

diff --git a/config/gemma4_doa_punctuation.yaml b/config/gemma4_doa_punctuation.yaml
deleted file mode 100644
index d49029d..0000000
--- a/config/gemma4_doa_punctuation.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-type: "simulstream.server.speech_processors.gemma4_doa.Gemma4DOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 60  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-attn_head: null  # Optional specific head; null means average over heads
-average_attn_over_layers: True
-cutoff_frame_num: __FRAME__
-detokenizer_type: "hf"
-hf_model_name: "google/gemma-4-E2B-it"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/gemma4_doa.py b/simulstream/server/speech_processors/gemma4_doa.py
deleted file mode 100644
index b834ec0..0000000
--- a/simulstream/server/speech_processors/gemma4_doa.py
+++ /dev/null
@@ -1,184 +0,0 @@
-# Copyright 2026 FBK
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-
-import logging
-from types import SimpleNamespace
-from typing import List, Tuple
-
-import numpy as np
-import torch
-
-from transformers import AutoModelForMultimodalLM, AutoProcessor
-
-from simulstream.server.speech_processors import class_load
-from simulstream.server.speech_processors.base_doa import (
-    DecoderOnlyAttention,
-    LANG_MAPPER,
-    TEMPLATED_SPEECH_PROMPT,
-)
-
-from transformers import set_seed
-torch.manual_seed(42)
-set_seed(42)
-
-
-logger = logging.getLogger(__name__)
-
-
-class Gemma4DOA(DecoderOnlyAttention):
-    """
-    Decoder-Only Attention agent for ``google/gemma-4-E2B-it`` and
-    ``google/gemma-4-E4B-it`` (the only Gemma 4 variants with audio support).
-
-    Extra config fields
-    -------------------
-    hf_model_name : str
-        Default: ``"google/gemma-4-E2B-it"``.
-    repetition_penalty : float
-        Repetition penalty for text generation. Default: ``1.0``.
-    no_repeat_ngram_size : int
-        N-gram blocking size. Default: ``0``.
-    """
-
-    BOW_PREFIX = " "
-    AUDIO_TOKEN_INDEX = 258881  # <|audio|>
-    AUDIO_TOKEN_STRIDE = 640    # 16000 / 25 tokens per second
-    GLOBAL_LAYERS = [4, 9, 14, 19, 24, 29, 34] # Indices of global (non-sliding) attention layers
-
-    def __init__(self, config: SimpleNamespace):
-        super().__init__(config)
-        self.bow_prefix = self.BOW_PREFIX
-        text_history_cls = class_load(self.text_history_config.type)
-        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
-        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
-        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
-        self.temperature = getattr(self.config, "temperature", 1.0)
-        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
-
-    @classmethod
-    def load_model(cls, config: SimpleNamespace) -> None:
-        model_name = getattr(
-            config,
-            "hf_model_name",
-            getattr(config, "model_path", "google/gemma-4-E2B-it"),
-        )
-        attn_impl = getattr(config, "attn_implementation", "eager")
-
-        cls.processor = AutoProcessor.from_pretrained(model_name)
-        cls.model = AutoModelForMultimodalLM.from_pretrained(
-            model_name,
-            torch_dtype="auto",
-            device_map="auto",
-            attn_implementation=attn_impl,
-        )
-        cls.model.eval()
-
-    def build_prompt(self) -> str:
-        return (
-            TEMPLATED_SPEECH_PROMPT
-            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
-            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
-        )
-
-    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        prefix = self.build_raw_text_prefix()
-
-        conversation = [
-            {
-                "role": "user",
-                "content": [
-                    {"type": "audio", "audio": waveform},
-                    {"type": "text", "text": self.build_prompt()},
-                ],
-            },
-        ]
-
-        inputs = self.processor.apply_chat_template(
-            conversation,
-            tokenize=True,
-            return_dict=True,
-            return_tensors="pt",
-            add_generation_prompt=True,
-            enable_thinking=False,
-        )
-
-        if prefix:
-            prefix_ids = self.processor.tokenizer(
-                prefix,
-                return_tensors="pt",
-                add_special_tokens=False,
-            ).input_ids
-            inputs["input_ids"] = torch.cat([inputs["input_ids"], prefix_ids], dim=1)
-            inputs["attention_mask"] = torch.cat(
-                [inputs["attention_mask"], torch.ones_like(prefix_ids)], dim=1
-            )
-
-        return {
-            k: v.to(self.device) if isinstance(v, torch.Tensor) else v
-            for k, v in inputs.items()
-        }
-
-    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
-        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
-
-    def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
-        return torch.stack(
-            [self._select_attn_from_layer(step_attn[i]) for i in self.GLOBAL_LAYERS],
-            dim=0,
-        ).mean(dim=0)
-
-    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
-        input_ids = inputs["input_ids"]
-        input_len = input_ids.shape[1]
-
-        audio_positions = self._find_audio_positions(input_ids)
-        audio_len = audio_positions.shape[0]
-
-        output = self.model.generate(
-            **inputs,
-            max_new_tokens=self.max_new_tokens,
-            repetition_penalty=self.repetition_penalty,
-            no_repeat_ngram_size=self.no_repeat_ngram_size,
-            output_attentions=True,
-            return_dict_in_generate=True,
-            do_sample=False,
-            temperature=self.temperature,
-        )
-
-        new_ids = output.sequences[:, input_len:]
-        new_tokens = [
-            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in new_ids[0]
-        ]
-
-        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        prefix_len = len(self.text_history) if self.text_history else 0
-        empty_attn = torch.zeros(0, audio_len, device=self.device)
-
-        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
-            if prefix_len > 0 else empty_attn
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
-        new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:]
-        ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-
-        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
-        cross_attn = self.normalize_attn(cross_attn)
-        return new_tokens, cross_attn
-
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
\ No newline at end of file

From 5a437db8c7e3801a8793b93ba05ac5058a1b9eb0 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 17:20:33 +0200
Subject: [PATCH 118/157] Revert "seed"

This reverts commit 96128c35f808d7e699ae34a202c81d147b833c1a.
---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 0712d20..72363a3 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -24,8 +24,8 @@
 from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, LANG_MAPPER
 
 from transformers import set_seed
-torch.manual_seed(41)
-set_seed(41)
+torch.manual_seed(42)
+set_seed(42)
 
 
 class Phi4MultimodalDOA(DecoderOnlyAttention):

From a9dcd6b7fc034fddc43384424e799bcfa7caa185 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 19:20:34 +0200
Subject: [PATCH 119/157] Add UltraVox

---
 config/ultravox_doa_punctuation.yaml          |  14 ++
 .../server/speech_processors/ultravox_doa.py  | 186 ++++++++++++++++++
 2 files changed, 200 insertions(+)
 create mode 100644 config/ultravox_doa_punctuation.yaml
 create mode 100644 simulstream/server/speech_processors/ultravox_doa.py

diff --git a/config/ultravox_doa_punctuation.yaml b/config/ultravox_doa_punctuation.yaml
new file mode 100644
index 0000000..e976e7a
--- /dev/null
+++ b/config/ultravox_doa_punctuation.yaml
@@ -0,0 +1,14 @@
+type: "simulstream.server.speech_processors.ultravox_doa.UltravoxDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 90  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+attn_head: null  # Optional specific head; null means average over heads
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
+detokenizer_type: "hf"
+hf_model_name: "fixie-ai/ultravox-v0_6-llama-3_1-8b"e
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
diff --git a/simulstream/server/speech_processors/ultravox_doa.py b/simulstream/server/speech_processors/ultravox_doa.py
new file mode 100644
index 0000000..3b401a4
--- /dev/null
+++ b/simulstream/server/speech_processors/ultravox_doa.py
@@ -0,0 +1,186 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import logging
+from types import SimpleNamespace
+from typing import List, Tuple
+
+import numpy as np
+import torch
+import transformers
+
+from simulstream.server.speech_processors import SAMPLE_RATE, class_load
+from simulstream.server.speech_processors.base_doa import (
+    DecoderOnlyAttention,
+    LANG_MAPPER,
+    TEMPLATED_SPEECH_PROMPT,
+)
+
+from transformers import set_seed
+torch.manual_seed(42)
+set_seed(42)
+
+
+logger = logging.getLogger(__name__)
+
+
+class UltravoxDOA(DecoderOnlyAttention):
+    """
+    Decoder-Only Attention agent for UltraVox.
+
+    Architecture: Whisper-large-v3-turbo encoder → stack_factor=8 projector → Llama-3.1-8B.
+    Audio is injected into the LLM embeddings at ``<|audio|>`` placeholder positions.
+    The pipeline preprocessor handles tokenization and audio feature extraction.
+
+    Extra config fields
+    -------------------
+    hf_model_name : str
+        Default: ``"fixie-ai/ultravox-v0_6-llama-3_1-8b"``.
+    repetition_penalty : float
+        Default: ``1.0``.
+    no_repeat_ngram_size : int
+        Default: ``0``.
+    """
+
+    BOW_PREFIX = " "
+    # Whisper-large-v3-turbo: 50 frames/s; stack_factor=8 → 50/8 tokens/s
+    # stride = 16000 / (50/8) = 2560 samples per audio token
+    AUDIO_TOKEN_STRIDE = 2560
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.bow_prefix = self.BOW_PREFIX
+        text_history_cls = class_load(self.text_history_config.type)
+        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
+        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
+        self.temperature = getattr(self.config, "temperature", 1.0)
+        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
+
+    @classmethod
+    def load_model(cls, config: SimpleNamespace) -> None:
+        model_name = getattr(
+            config,
+            "hf_model_name",
+            getattr(config, "model_path", "fixie-ai/ultravox-v0_6-llama-3_1-8b"),
+        )
+        attn_impl = getattr(config, "attn_implementation", "eager")
+
+        cls.pipe = transformers.pipeline(
+            model=model_name,
+            trust_remote_code=True,
+            torch_dtype=torch.bfloat16,
+            device_map="auto",
+            model_kwargs={"attn_implementation": attn_impl},
+        )
+        cls.model = cls.pipe.model
+        cls.model.eval()
+
+    def build_prompt(self) -> str:
+        return (
+            TEMPLATED_SPEECH_PROMPT
+            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
+            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
+        )
+
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        prefix = self.build_raw_text_prefix()
+        turns = [
+            {
+                "role": "system",
+                "content": self.build_prompt(),
+            },
+        ]
+        # The pipeline preprocessor tokenizes the conversation, extracts audio
+        # features, and returns audio_token_start_idx + audio_token_len alongside
+        # the standard input_ids/attention_mask/audio_values tensors.
+        inputs = self.pipe.preprocess(
+            {"audio": waveform, "turns": turns, "sampling_rate": SAMPLE_RATE}
+        )
+
+        if prefix:
+            prefix_ids = self.pipe.tokenizer(
+                prefix,
+                return_tensors="pt",
+                add_special_tokens=False,
+            ).input_ids
+            inputs["input_ids"] = torch.cat([inputs["input_ids"], prefix_ids], dim=1)
+            inputs["attention_mask"] = torch.cat(
+                [inputs["attention_mask"], torch.ones_like(prefix_ids)], dim=1
+            )
+
+        return {
+            k: v.to(self.device) if isinstance(v, torch.Tensor) else v
+            for k, v in inputs.items()
+        }
+
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        # Ultravox pipeline provides audio_token_start_idx and audio_token_len
+        # directly — no need to scan input_ids for a placeholder token.
+        # Called with the full inputs dict via _generate.
+        raise NotImplementedError("Use _find_audio_positions_from_inputs instead.")
+
+    def _find_audio_positions_from_inputs(self, inputs: dict) -> torch.Tensor:
+        start = inputs["audio_token_start_idx"][0].item()
+        length = inputs["audio_token_len"][0].item()
+        return torch.arange(start, start + length, device=self.device)
+
+    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+        input_ids = inputs["input_ids"]
+        input_len = input_ids.shape[1]
+
+        audio_positions = self._find_audio_positions_from_inputs(inputs)
+        audio_len = audio_positions.shape[0]
+
+        output = self.model.generate(
+            **inputs,
+            max_new_tokens=self.max_new_tokens,
+            repetition_penalty=self.repetition_penalty,
+            no_repeat_ngram_size=self.no_repeat_ngram_size,
+            output_attentions=True,
+            return_dict_in_generate=True,
+            do_sample=False,
+            temperature=self.temperature,
+            eos_token_id=[
+                self.pipe.tokenizer.eos_token_id,
+                self.pipe.tokenizer.convert_tokens_to_ids("<|eot_id|>"),
+            ],
+        )
+
+        new_ids = output.sequences[:, input_len:]
+        new_tokens = [
+            self.pipe.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in new_ids[0]
+        ]
+
+        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
+        prefix_len = len(self.text_history) if self.text_history else 0
+        empty_attn = torch.zeros(0, audio_len, device=self.device)
+
+        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
+            if prefix_len > 0 else empty_attn
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
+        new_rows = [
+            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:]
+        ]
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
+        cross_attn = self.normalize_attn(cross_attn)
+        return new_tokens, cross_attn
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens)
\ No newline at end of file

From af80da4e9c6c7bcce569577c15a45fcebc578644 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 19:24:10 +0200
Subject: [PATCH 120/157] Add UltraVox

---
 config/ultravox_doa_punctuation.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/ultravox_doa_punctuation.yaml b/config/ultravox_doa_punctuation.yaml
index e976e7a..8d6f972 100644
--- a/config/ultravox_doa_punctuation.yaml
+++ b/config/ultravox_doa_punctuation.yaml
@@ -1,7 +1,7 @@
 type: "simulstream.server.speech_processors.ultravox_doa.UltravoxDOA"
 text_history:
   type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 90  # Maximum length for the audio buffer, in seconds
+audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
 text_history_max_len: 128
 speech_chunk_size: 1  # seconds
 attn_layer: __LAYER__
@@ -11,4 +11,4 @@ cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
 hf_model_name: "fixie-ai/ultravox-v0_6-llama-3_1-8b"e
 word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
+max_new_tokens: 32
\ No newline at end of file

From d89741cabcdca4b2edd4d541a0c101a0c28ad9a7 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 21 May 2026 19:28:55 +0200
Subject: [PATCH 121/157] Add UltraVox

---
 config/ultravox_doa_punctuation.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/ultravox_doa_punctuation.yaml b/config/ultravox_doa_punctuation.yaml
index 8d6f972..61e7913 100644
--- a/config/ultravox_doa_punctuation.yaml
+++ b/config/ultravox_doa_punctuation.yaml
@@ -9,6 +9,6 @@ attn_head: null  # Optional specific head; null means average over heads
 average_attn_over_layers: True
 cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
-hf_model_name: "fixie-ai/ultravox-v0_6-llama-3_1-8b"e
+hf_model_name: "fixie-ai/ultravox-v0_6-llama-3_1-8b"
 word_level_postprocess: True  # Disable if character-level language
 max_new_tokens: 32
\ No newline at end of file

From 6c1ceadb5ea8d9b05d56f0e950fe651eb9a18f6e Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 23 May 2026 13:03:50 +0200
Subject: [PATCH 122/157] Comment attn implementation

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 72363a3..2508e8e 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -60,7 +60,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
             device_map="cuda",
             torch_dtype="auto",
             trust_remote_code=True,
-            _attn_implementation="eager",
+            #_attn_implementation="eager",
         )
         cls.model.eval()
         cls.generation_config = GenerationConfig.from_pretrained(model_path)

From db80adaee8ea8ea154e9359cc36024f2f08ba14f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 23 May 2026 13:28:15 +0200
Subject: [PATCH 123/157] Revert "Comment attn implementation"

This reverts commit 6c1ceadb5ea8d9b05d56f0e950fe651eb9a18f6e.
---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 2508e8e..72363a3 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -60,7 +60,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
             device_map="cuda",
             torch_dtype="auto",
             trust_remote_code=True,
-            #_attn_implementation="eager",
+            _attn_implementation="eager",
         )
         cls.model.eval()
         cls.generation_config = GenerationConfig.from_pretrained(model_path)

From 41ba86f1b0ad8808052c3f5639f3f6ebced1e2c2 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 23 May 2026 14:57:01 +0200
Subject: [PATCH 124/157] Change Qwen2Audio prompt

---
 simulstream/server/speech_processors/qwenaudio_doa.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
index 3936856..0785ddc 100644
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ b/simulstream/server/speech_processors/qwenaudio_doa.py
@@ -90,11 +90,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
         cls.model.eval()
 
     def build_prompt(self) -> str:
-        return (
-            TEMPLATED_SPEECH_PROMPT
-            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
-            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
-        )
+        return f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}:"
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         prefix = self.build_raw_text_prefix()

From d5020a0c9588f7758c26fe60710c109976a8bf4d Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 23 May 2026 18:44:14 +0200
Subject: [PATCH 125/157] Add Voxtral

---
 config/voxtral_doa_punctuation.yaml           |  14 ++
 .../server/speech_processors/voxtral_doa.py   | 168 ++++++++++++++++++
 2 files changed, 182 insertions(+)
 create mode 100644 config/voxtral_doa_punctuation.yaml
 create mode 100644 simulstream/server/speech_processors/voxtral_doa.py

diff --git a/config/voxtral_doa_punctuation.yaml b/config/voxtral_doa_punctuation.yaml
new file mode 100644
index 0000000..eb1efba
--- /dev/null
+++ b/config/voxtral_doa_punctuation.yaml
@@ -0,0 +1,14 @@
+type: "simulstream.server.speech_processors.voxtral_doa.VoxtralDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
+text_history_max_len: 128
+speech_chunk_size: 1  # seconds
+attn_layer: __LAYER__
+attn_head: null  # Optional specific head; null means average over heads
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
+detokenizer_type: "hf"
+hf_model_name: "mistralai/Voxtral-Mini-3B-2507"
+word_level_postprocess: True  # Disable if character-level language
+max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/voxtral_doa.py b/simulstream/server/speech_processors/voxtral_doa.py
new file mode 100644
index 0000000..91b7b66
--- /dev/null
+++ b/simulstream/server/speech_processors/voxtral_doa.py
@@ -0,0 +1,168 @@
+# Copyright 2026 FBK
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+import logging
+import tempfile
+import os
+from types import SimpleNamespace
+from typing import List, Tuple
+
+import numpy as np
+import soundfile as sf
+import torch
+
+from transformers import AutoProcessor, VoxtralForConditionalGeneration
+
+from simulstream.server.speech_processors import SAMPLE_RATE, class_load
+from simulstream.server.speech_processors.base_doa import (
+    DecoderOnlyAttention,
+    LANG_MAPPER,
+    TEMPLATED_SPEECH_PROMPT,
+)
+
+from transformers import set_seed
+torch.manual_seed(42)
+set_seed(42)
+
+
+logger = logging.getLogger(__name__)
+
+
+class VoxtralDOA(DecoderOnlyAttention):
+    """
+    Decoder-Only Attention agent for ``mistralai/Voxtral-Mini-3B-2507`` and
+    ``mistralai/Voxtral-Small-24B-2507``.
+
+    Extra config fields
+    -------------------
+    hf_model_name : str
+        Default: ``"mistralai/Voxtral-Mini-3B-2507"``.
+    repetition_penalty : float
+        Default: ``1.0``.
+    no_repeat_ngram_size : int
+        Default: ``0``.
+    """
+
+    BOW_PREFIX = " "
+    # To be resolved from model config at load_model time
+    AUDIO_TOKEN_INDEX = None
+    # Voxtral encoder: 50 frames/s, stride 4 → 12.5 tokens/s
+    # stride = 16000 / 12.5 = 1280 samples per audio token
+    AUDIO_TOKEN_STRIDE = 1280
+
+    def __init__(self, config: SimpleNamespace):
+        super().__init__(config)
+        self.bow_prefix = self.BOW_PREFIX
+        text_history_cls = class_load(self.text_history_config.type)
+        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
+        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
+        self.temperature = getattr(self.config, "temperature", 1.0)
+        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
+
+    @classmethod
+    def load_model(cls, config: SimpleNamespace) -> None:
+        model_name = getattr(
+            config,
+            "hf_model_name",
+            getattr(config, "model_path", "mistralai/Voxtral-Mini-3B-2507"),
+        )
+        attn_impl = getattr(config, "attn_implementation", "eager")
+
+        cls.processor = AutoProcessor.from_pretrained(model_name)
+        cls.model = VoxtralForConditionalGeneration.from_pretrained(
+            model_name,
+            torch_dtype=torch.bfloat16,
+            device_map="auto",
+            attn_implementation=attn_impl,
+        )
+        cls.model.eval()
+        cls.AUDIO_TOKEN_INDEX = cls.model.config.audio_token_id
+
+    def build_prompt(self) -> str:
+        return (
+            TEMPLATED_SPEECH_PROMPT
+            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
+            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
+        )
+
+    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        prefix = self.build_raw_text_prefix()
+
+        # apply_chat_template requires a file path — write waveform to a temp file
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+            tmp_path = f.name
+        sf.write(tmp_path, waveform, SAMPLE_RATE)
+
+        try:
+            conversation = {
+                "role": "user",
+                "content": [
+                    {"type": "audio", "path": tmp_path},
+                    {"type": "text", "text": f"{self.build_prompt()}{prefix}"},
+                ],
+            }
+            inputs = self.processor.apply_chat_template([conversation])
+        finally:
+            os.unlink(tmp_path)
+
+        return inputs.to(self.device, dtype=torch.bfloat16)
+
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
+
+    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+        input_ids = inputs["input_ids"]
+        input_len = input_ids.shape[1]
+
+        audio_positions = self._find_audio_positions(input_ids)
+        audio_len = audio_positions.shape[0]
+
+        output = self.model.generate(
+            **inputs,
+            max_new_tokens=self.max_new_tokens,
+            repetition_penalty=self.repetition_penalty,
+            no_repeat_ngram_size=self.no_repeat_ngram_size,
+            output_attentions=True,
+            return_dict_in_generate=True,
+            do_sample=False,
+            temperature=self.temperature,
+        )
+
+        new_ids = output.sequences[:, input_len:]
+        new_tokens = [
+            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
+            for token_id in new_ids[0]
+        ]
+
+        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
+        prefix_len = len(self.text_history) if self.text_history else 0
+        empty_attn = torch.zeros(0, audio_len, device=self.device)
+
+        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
+            if prefix_len > 0 else empty_attn
+        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
+        new_rows = [
+            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
+            for step_attn in output.attentions[1:]
+        ]
+        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
+        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
+        cross_attn = self.normalize_attn(cross_attn)
+        return new_tokens, cross_attn
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens)
\ No newline at end of file

From e89a3f50e2b643d87a66ac24f70abc453fe82e53 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sat, 23 May 2026 20:00:33 +0200
Subject: [PATCH 126/157] Add Voxtral

---
 .../server/speech_processors/voxtral_doa.py   | 33 ++++++++++++-------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/simulstream/server/speech_processors/voxtral_doa.py b/simulstream/server/speech_processors/voxtral_doa.py
index 91b7b66..1673b47 100644
--- a/simulstream/server/speech_processors/voxtral_doa.py
+++ b/simulstream/server/speech_processors/voxtral_doa.py
@@ -98,25 +98,36 @@ def build_prompt(self) -> str:
         )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        prefix = self.build_raw_text_prefix()
-
-        # apply_chat_template requires a file path — write waveform to a temp file
         with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
             tmp_path = f.name
         sf.write(tmp_path, waveform, SAMPLE_RATE)
 
         try:
-            conversation = {
-                "role": "user",
-                "content": [
-                    {"type": "audio", "path": tmp_path},
-                    {"type": "text", "text": f"{self.build_prompt()}{prefix}"},
-                ],
-            }
-            inputs = self.processor.apply_chat_template([conversation])
+            conversation = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "audio", "path": tmp_path},
+                        {"type": "text", "text": self.build_prompt()},
+                    ],
+                },
+            ]
+            inputs = self.processor.apply_chat_template(conversation)
         finally:
             os.unlink(tmp_path)
 
+        prefix = self.build_raw_text_prefix()
+        if prefix:
+            prefix_ids = self.processor.tokenizer(
+                prefix,
+                return_tensors="pt",
+                add_special_tokens=False,
+            ).input_ids
+            inputs["input_ids"] = torch.cat([inputs["input_ids"], prefix_ids], dim=1)
+            inputs["attention_mask"] = torch.cat(
+                [inputs["attention_mask"], torch.ones_like(prefix_ids)], dim=1
+            )
+
         return inputs.to(self.device, dtype=torch.bfloat16)
 
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:

From a709c745d55d99002a669d7c81201ac1847b30d8 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 24 May 2026 12:35:48 +0200
Subject: [PATCH 127/157] Add Voxtral

---
 .../server/speech_processors/voxtral_doa.py   | 35 +++++++++----------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/simulstream/server/speech_processors/voxtral_doa.py b/simulstream/server/speech_processors/voxtral_doa.py
index 1673b47..54ed132 100644
--- a/simulstream/server/speech_processors/voxtral_doa.py
+++ b/simulstream/server/speech_processors/voxtral_doa.py
@@ -13,8 +13,8 @@
 # limitations under the License
 
 import logging
-import tempfile
-import os
+import base64
+import io
 from types import SimpleNamespace
 from typing import List, Tuple
 
@@ -98,23 +98,20 @@ def build_prompt(self) -> str:
         )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
-            tmp_path = f.name
-        sf.write(tmp_path, waveform, SAMPLE_RATE)
-
-        try:
-            conversation = [
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "audio", "path": tmp_path},
-                        {"type": "text", "text": self.build_prompt()},
-                    ],
-                },
-            ]
-            inputs = self.processor.apply_chat_template(conversation)
-        finally:
-            os.unlink(tmp_path)
+        audio_buffer = io.BytesIO()
+        sf.write(audio_buffer, waveform, SAMPLE_RATE, format="WAV")
+        audio_base64 = base64.b64encode(audio_buffer.getvalue()).decode("utf-8")
+
+        conversation = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "audio", "base64": audio_base64},
+                    {"type": "text", "text": self.build_prompt()},
+                ],
+            },
+        ]
+        inputs = self.processor.apply_chat_template(conversation)
 
         prefix = self.build_raw_text_prefix()
         if prefix:

From ffcf1c4643d2a708ad69eaaa4a78de02a3a9d305 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 24 May 2026 14:20:06 +0200
Subject: [PATCH 128/157] Fix tokenizer

---
 simulstream/metrics/detokenizers.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index defdadd..83ede51 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -24,8 +24,13 @@ def build_hf_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
     processor = AutoProcessor.from_pretrained(config.hf_model_name, trust_remote_code=True)
     tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
 
-    def detokenize(input_tokens: List[str]) -> str:
-        return tokenizer.convert_tokens_to_string(input_tokens)
+    if hasattr(tokenizer, "convert_tokens_to_string"):
+        def detokenize(input_tokens: List[str]) -> str:
+            return tokenizer.convert_tokens_to_string(input_tokens)
+    else:
+        def detokenize(input_tokens: List[str]) -> str:
+            ids = tokenizer.convert_tokens_to_ids(input_tokens)
+            return tokenizer.tokenizer.decode(ids, skip_special_tokens=True)
 
     return detokenize
 

From c0f49953f50f9615c38ebda34bfcd81319408a92 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 24 May 2026 14:24:32 +0200
Subject: [PATCH 129/157] Fix voxtral tokenizer

---
 config/voxtral_doa_punctuation.yaml |  2 +-
 simulstream/metrics/detokenizers.py | 27 +++++++++++++++++++--------
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/config/voxtral_doa_punctuation.yaml b/config/voxtral_doa_punctuation.yaml
index eb1efba..780f051 100644
--- a/config/voxtral_doa_punctuation.yaml
+++ b/config/voxtral_doa_punctuation.yaml
@@ -8,7 +8,7 @@ attn_layer: __LAYER__
 attn_head: null  # Optional specific head; null means average over heads
 average_attn_over_layers: True
 cutoff_frame_num: __FRAME__
-detokenizer_type: "hf"
+detokenizer_type: "voxtral"
 hf_model_name: "mistralai/Voxtral-Mini-3B-2507"
 word_level_postprocess: True  # Disable if character-level language
 max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 83ede51..1fa5dba 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -24,13 +24,23 @@ def build_hf_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
     processor = AutoProcessor.from_pretrained(config.hf_model_name, trust_remote_code=True)
     tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
 
-    if hasattr(tokenizer, "convert_tokens_to_string"):
-        def detokenize(input_tokens: List[str]) -> str:
-            return tokenizer.convert_tokens_to_string(input_tokens)
-    else:
-        def detokenize(input_tokens: List[str]) -> str:
-            ids = tokenizer.convert_tokens_to_ids(input_tokens)
-            return tokenizer.tokenizer.decode(ids, skip_special_tokens=True)
+    def detokenize(input_tokens: List[str]) -> str:
+        return tokenizer.convert_tokens_to_string(input_tokens)
+
+    return detokenize
+
+def build_voxtral_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
+    from transformers import AutoProcessor
+
+    assert hasattr(config, "hf_model_name"), \
+        "`hf_model_name` required in the eval config for `voxtral` detokenizer"
+    processor = AutoProcessor.from_pretrained(config.hf_model_name)
+    # MistralCommonTokenizer wraps the actual tokenizer under .tokenizer
+    tokenizer = processor.tokenizer.tokenizer
+
+    def detokenize(input_tokens: List[str]) -> str:
+        ids = tokenizer.convert_tokens_to_ids(input_tokens)
+        return tokenizer.decode(ids, skip_special_tokens=True)
 
     return detokenize
 
@@ -69,7 +79,8 @@ def detokenize(input_tokens: List[str]) -> str:
 _DETOKENIZER_BUILDER_MAP: Dict[str, Callable[[SimpleNamespace], Callable[[List[str]], str]]] = {
     "hf": build_hf_detokenizer,
     "canary": build_canary_detokenizer,
-    "simuleval": build_simuleval_detokenizer
+    "simuleval": build_simuleval_detokenizer,
+    "voxtral": build_voxtral_detokenizer,
 }
 
 

From 7b34fe551cec31a44c53f9df602553b30b92732c Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 24 May 2026 14:26:45 +0200
Subject: [PATCH 130/157] Fix voxtral tokenizer

---
 simulstream/metrics/detokenizers.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 1fa5dba..4c48344 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -35,8 +35,7 @@ def build_voxtral_detokenizer(config: SimpleNamespace) -> Callable[[List[str]],
     assert hasattr(config, "hf_model_name"), \
         "`hf_model_name` required in the eval config for `voxtral` detokenizer"
     processor = AutoProcessor.from_pretrained(config.hf_model_name)
-    # MistralCommonTokenizer wraps the actual tokenizer under .tokenizer
-    tokenizer = processor.tokenizer.tokenizer
+    tokenizer = processor.audio_tokenizer
 
     def detokenize(input_tokens: List[str]) -> str:
         ids = tokenizer.convert_tokens_to_ids(input_tokens)

From a2d00da48372e552b4adfaa98f70ba9ba25bdd5a Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 24 May 2026 14:32:50 +0200
Subject: [PATCH 131/157] Fix voxtral tokenizer

---
 simulstream/metrics/detokenizers.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 4c48344..3cd5b15 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -30,16 +30,16 @@ def detokenize(input_tokens: List[str]) -> str:
     return detokenize
 
 def build_voxtral_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
-    from transformers import AutoProcessor
+    from transformers import AutoTokenizer
 
     assert hasattr(config, "hf_model_name"), \
         "`hf_model_name` required in the eval config for `voxtral` detokenizer"
-    processor = AutoProcessor.from_pretrained(config.hf_model_name)
-    tokenizer = processor.audio_tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(config.hf_model_name)
 
     def detokenize(input_tokens: List[str]) -> str:
-        ids = tokenizer.convert_tokens_to_ids(input_tokens)
-        return tokenizer.decode(ids, skip_special_tokens=True)
+        text = "".join(input_tokens)
+        ids = tokenizer.encode(text, bos=False, eos=False)
+        return tokenizer.tokenizer.decode(ids)
 
     return detokenize
 

From 538688194a1d9cfb95e3ef9a17b38578a29c8655 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 24 May 2026 14:36:21 +0200
Subject: [PATCH 132/157] Fix voxtral tokenizer

---
 simulstream/metrics/detokenizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 3cd5b15..21e4b2d 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -38,7 +38,7 @@ def build_voxtral_detokenizer(config: SimpleNamespace) -> Callable[[List[str]],
 
     def detokenize(input_tokens: List[str]) -> str:
         text = "".join(input_tokens)
-        ids = tokenizer.encode(text, bos=False, eos=False)
+        ids = tokenizer.encode(text, add_special_tokens=False)
         return tokenizer.tokenizer.decode(ids)
 
     return detokenize

From 278786f884704c2e6d2cfc7eebbba67a63bffcfc Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Sun, 24 May 2026 15:26:55 +0200
Subject: [PATCH 133/157] Fix voxtral prompt

---
 simulstream/server/speech_processors/voxtral_doa.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/voxtral_doa.py b/simulstream/server/speech_processors/voxtral_doa.py
index 54ed132..55ed67f 100644
--- a/simulstream/server/speech_processors/voxtral_doa.py
+++ b/simulstream/server/speech_processors/voxtral_doa.py
@@ -28,7 +28,6 @@
 from simulstream.server.speech_processors.base_doa import (
     DecoderOnlyAttention,
     LANG_MAPPER,
-    TEMPLATED_SPEECH_PROMPT,
 )
 
 from transformers import set_seed
@@ -36,6 +35,13 @@
 set_seed(42)
 
 
+SUGGESTED_PROMPT = \
+    ("You are an expert multilingual speech translator. Your task is to accurately translate "
+     "the provided audio from {src_lang} into {tgt_lang}. Output ONLY the translated text. "
+     "Maintain the original tone, context, and speaker intent without adding any extra "
+     "conversational filler.")
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -92,7 +98,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
 
     def build_prompt(self) -> str:
         return (
-            TEMPLATED_SPEECH_PROMPT
+            SUGGESTED_PROMPT
             .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
             .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
         )

From d124275238b0ab573bd27547e923635f4a911242 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 28 May 2026 14:43:16 +0200
Subject: [PATCH 134/157] Clean for release

---
 config/qwen2audio_doa_punctuation.yaml        |  15 --
 config/qwen3omni_doa_punctuation.yaml         |   1 -
 config/ultravox_doa_punctuation.yaml          |  14 --
 config/voxtral_doa_punctuation.yaml           |  14 --
 .../server/speech_processors/base_doa.py      |  10 +-
 .../speech_processors/phi4multimodal_doa.py   |   5 +-
 .../server/speech_processors/qwenaudio_doa.py | 170 ----------------
 .../server/speech_processors/qwenomni_doa.py  |   7 +-
 .../server/speech_processors/ultravox_doa.py  | 186 ------------------
 .../server/speech_processors/voxtral_doa.py   | 182 -----------------
 10 files changed, 11 insertions(+), 593 deletions(-)
 delete mode 100644 config/qwen2audio_doa_punctuation.yaml
 delete mode 100644 config/ultravox_doa_punctuation.yaml
 delete mode 100644 config/voxtral_doa_punctuation.yaml
 delete mode 100644 simulstream/server/speech_processors/qwenaudio_doa.py
 delete mode 100644 simulstream/server/speech_processors/ultravox_doa.py
 delete mode 100644 simulstream/server/speech_processors/voxtral_doa.py

diff --git a/config/qwen2audio_doa_punctuation.yaml b/config/qwen2audio_doa_punctuation.yaml
deleted file mode 100644
index cadad1b..0000000
--- a/config/qwen2audio_doa_punctuation.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-type: "simulstream.server.speech_processors.qwenaudio_doa.Qwen2AudioDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-attn_head: null  # Optional specific head; null means average over heads
-average_attn_over_layers: True
-cutoff_frame_num: __FRAME__
-detokenizer_type: "hf"
-hf_model_name: "Qwen/Qwen2-Audio-7B-Instruct"
-use_video: False
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
diff --git a/config/qwen3omni_doa_punctuation.yaml b/config/qwen3omni_doa_punctuation.yaml
index 443f4bc..96e6f02 100644
--- a/config/qwen3omni_doa_punctuation.yaml
+++ b/config/qwen3omni_doa_punctuation.yaml
@@ -10,6 +10,5 @@ average_attn_over_layers: True
 cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
 hf_model_name: "Qwen/Qwen3-Omni-30B-A3B-Instruct"
-use_video: False
 word_level_postprocess: True  # Disable if character-level language
 max_new_tokens: 32
\ No newline at end of file
diff --git a/config/ultravox_doa_punctuation.yaml b/config/ultravox_doa_punctuation.yaml
deleted file mode 100644
index 61e7913..0000000
--- a/config/ultravox_doa_punctuation.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-type: "simulstream.server.speech_processors.ultravox_doa.UltravoxDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-attn_head: null  # Optional specific head; null means average over heads
-average_attn_over_layers: True
-cutoff_frame_num: __FRAME__
-detokenizer_type: "hf"
-hf_model_name: "fixie-ai/ultravox-v0_6-llama-3_1-8b"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
diff --git a/config/voxtral_doa_punctuation.yaml b/config/voxtral_doa_punctuation.yaml
deleted file mode 100644
index 780f051..0000000
--- a/config/voxtral_doa_punctuation.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-type: "simulstream.server.speech_processors.voxtral_doa.VoxtralDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 120  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-attn_head: null  # Optional specific head; null means average over heads
-average_attn_over_layers: True
-cutoff_frame_num: __FRAME__
-detokenizer_type: "voxtral"
-hf_model_name: "mistralai/Voxtral-Mini-3B-2507"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 3274968..e541000 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -39,7 +39,7 @@
 
 class DecoderOnlyAttention(BaseStreamAtt):
     """
-    Generic Decoder-only Attention-based policy for SpeechLLMs.
+    Generic Decoder-only Attention-based (DOA) policy for SpeechLLMs.
 
     The class handles:
     - Raw-waveform history accumulation.
@@ -53,9 +53,9 @@ class DecoderOnlyAttention(BaseStreamAtt):
     ----------
     config : SimpleNamespace
         All fields from :class:`BaseStreamAtt`, plus:
-        attn_layer : int
+        cross_attn_layer : int
             Layer from which to extract attention scores. Default: ``0``.
-        attn_head : int | None
+        cross_attn_head : int | None
             Attention head to use. If ``None``, attention scores are averaged
             over all heads. If set together with
             ``average_attn_over_layers=True``, the selected head is averaged
@@ -67,6 +67,8 @@ class DecoderOnlyAttention(BaseStreamAtt):
         audio_history_max_duration : int
             Maximum raw waveform length to keep in the rolling history.
             Default: ``180`` (seconds).
+        device : torch.device
+            Device to use for model's loading and execution.
         max_new_tokens : int
             Maximum tokens to generate per chunk.  Default: ``32``.
 
@@ -88,7 +90,7 @@ def __init__(self, config: SimpleNamespace):
         super().__init__(config)
         self.cross_attn_layer = getattr(self.config, "attn_layer", 0)
         self.cross_attn_head = getattr(self.config, "attn_head", None)
-        self.average_attn_over_layers = getattr(self.config, "average_attn_over_layers", False)
+        self.average_attn_over_layers = getattr(self.config, "average_attn_over_layers", True)
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 72363a3..8930159 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -30,7 +30,7 @@
 
 class Phi4MultimodalDOA(DecoderOnlyAttention):
     """
-    Decoder-Only Attention agent for ``microsoft/Phi-4-multimodal-instruct``.
+    Decoder-Only Attention agent for Phi4-Multimodal.
     """
 
     # Phi-4 special tokens
@@ -45,9 +45,8 @@ class Phi4MultimodalDOA(DecoderOnlyAttention):
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
-        self.bow_prefix = self.BOW_PREFIX
         text_history_cls = class_load(self.text_history_config.type)
-        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.text_history_method = text_history_cls(self.text_history_config, self.BOW_PREFIX)
         self.audio_subsampling_factor = self.ENCODER_SUBSAMPLING_FACTOR * self.HOP_LENGTH
 
     @classmethod
diff --git a/simulstream/server/speech_processors/qwenaudio_doa.py b/simulstream/server/speech_processors/qwenaudio_doa.py
deleted file mode 100644
index 0785ddc..0000000
--- a/simulstream/server/speech_processors/qwenaudio_doa.py
+++ /dev/null
@@ -1,170 +0,0 @@
-# Copyright 2026 FBK
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-
-import logging
-from types import SimpleNamespace
-from typing import List, Tuple
-
-import numpy as np
-import torch
-
-from transformers import AutoProcessor, Qwen2AudioForConditionalGeneration
-
-from simulstream.server.speech_processors import SAMPLE_RATE, class_load
-from simulstream.server.speech_processors.base_doa import (
-    DecoderOnlyAttention,
-    LANG_MAPPER,
-    TEMPLATED_SPEECH_PROMPT,
-)
-
-from transformers import set_seed
-torch.manual_seed(42)
-set_seed(42)
-
-
-logger = logging.getLogger(__name__)
-
-
-class Qwen2AudioDOA(DecoderOnlyAttention):
-    """
-    Decoder-Only Attention agent for ``Qwen/Qwen2-Audio-7B-Instruct``.
-
-    Architecture: Whisper encoder (subsampling factor 2) → linear projector → Qwen2-7B.
-    Audio is serialized through the official Qwen2-Audio chat template and then
-    expanded by the processor into repeated ``<|AUDIO|>`` placeholder tokens in
-    ``input_ids``.
-
-    Extra config fields
-    -------------------
-    hf_model_name : str
-        Default: ``"Qwen/Qwen2-Audio-7B-Instruct"``.
-    repetition_penalty : float
-        Default: ``1.0``.
-    no_repeat_ngram_size : int
-        Default: ``0``.
-    """
-
-    BOW_PREFIX = " "
-    # Whisper encoder: 50 frames/s, subsampling factor 2 → 25 tokens/s
-    # stride = 16000 / 25 = 640 samples per audio token
-    AUDIO_TOKEN_STRIDE = 640
-    AUDIO_TOKEN_INDEX = 151646  # <|AUDIO|>
-
-    def __init__(self, config: SimpleNamespace):
-        super().__init__(config)
-        self.bow_prefix = self.BOW_PREFIX
-        text_history_cls = class_load(self.text_history_config.type)
-        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
-        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
-        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
-        self.temperature = getattr(self.config, "temperature", 1.0)
-        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
-
-    @classmethod
-    def load_model(cls, config: SimpleNamespace) -> None:
-        model_name = getattr(
-            config,
-            "hf_model_name",
-            getattr(config, "model_path", "Qwen/Qwen2-Audio-7B-Instruct"),
-        )
-        attn_impl = getattr(config, "attn_implementation", "eager")
-
-        cls.processor = AutoProcessor.from_pretrained(model_name)
-        cls.model = Qwen2AudioForConditionalGeneration.from_pretrained(
-            model_name,
-            torch_dtype="auto",
-            device_map="auto",
-            attn_implementation=attn_impl,
-        )
-        cls.model.eval()
-
-    def build_prompt(self) -> str:
-        return f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}:"
-
-    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        prefix = self.build_raw_text_prefix()
-
-        conversation = [
-            {"role": "system", "content": "You are a helpful assistant."},
-            {
-                "role": "user",
-                "content": [
-                    {"type": "audio", "audio_url": "placeholder"},
-                    {"type": "text", "text": self.build_prompt()},
-                ],
-            },
-        ]
-
-        text = self.processor.apply_chat_template(
-            conversation,
-            add_generation_prompt=True,
-            tokenize=False,
-        )
-
-        inputs = self.processor(
-            text=f"{text}{prefix}",
-            audio=[waveform],
-            return_tensors="pt",
-            padding=True,
-        )
-        return inputs.to(self.device)
-
-    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
-        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
-
-    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
-        input_ids = inputs["input_ids"]
-        input_len = input_ids.shape[1]
-
-        audio_positions = self._find_audio_positions(input_ids)
-        audio_len = audio_positions.shape[0]
-
-        output = self.model.generate(
-            **inputs,
-            max_new_tokens=self.max_new_tokens,
-            repetition_penalty=self.repetition_penalty,
-            no_repeat_ngram_size=self.no_repeat_ngram_size,
-            output_attentions=True,
-            return_dict_in_generate=True,
-            do_sample=False,
-            temperature=self.temperature,
-            eos_token_id=[151643, 151645],  # <|endoftext|> and <|im_end|>
-        )
-
-        new_ids = output.sequences[:, input_len:]
-        new_tokens = [
-            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in new_ids[0]
-        ]
-
-        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        prefix_len = len(self.text_history) if self.text_history else 0
-        empty_attn = torch.zeros(0, audio_len, device=self.device)
-
-        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
-            if prefix_len > 0 else empty_attn
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
-        new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:]
-        ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-
-        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
-        cross_attn = self.normalize_attn(cross_attn)
-        return new_tokens, cross_attn
-
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 6d14074..6ede630 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -39,14 +39,14 @@
 
 class Qwen3OmniDOA(DecoderOnlyAttention):
     """
-    Decoder-Only Attention agent for ``Qwen/Qwen3-Omni-*``.
+    Decoder-Only Attention agent for Qwen3-Omni.
 
     Extra config fields
     -------------------
-    hf_model_name : str
-        Default: ``"Qwen/Qwen3-Omni-30B-A3B-Instruct"``.
     repetition_penalty : float
         Repetition penalty for text generation. Default: ``1.05``.
+    temperature : float
+        Temperature for text generation. Default: ``1.0``.
     no_repeat_ngram_size : int
         N-gram blocking size for text generation. Default: ``5``.
     """
@@ -67,7 +67,6 @@ def __init__(self, config: SimpleNamespace):
         text_history_cls = class_load(self.text_history_config.type)
         self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
-        self.use_video = getattr(self.config, "use_video", False)
         self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.05)
         self.temperature = getattr(self.config, "temperature", 1.0)
         self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 5)
diff --git a/simulstream/server/speech_processors/ultravox_doa.py b/simulstream/server/speech_processors/ultravox_doa.py
deleted file mode 100644
index 3b401a4..0000000
--- a/simulstream/server/speech_processors/ultravox_doa.py
+++ /dev/null
@@ -1,186 +0,0 @@
-# Copyright 2026 FBK
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-
-import logging
-from types import SimpleNamespace
-from typing import List, Tuple
-
-import numpy as np
-import torch
-import transformers
-
-from simulstream.server.speech_processors import SAMPLE_RATE, class_load
-from simulstream.server.speech_processors.base_doa import (
-    DecoderOnlyAttention,
-    LANG_MAPPER,
-    TEMPLATED_SPEECH_PROMPT,
-)
-
-from transformers import set_seed
-torch.manual_seed(42)
-set_seed(42)
-
-
-logger = logging.getLogger(__name__)
-
-
-class UltravoxDOA(DecoderOnlyAttention):
-    """
-    Decoder-Only Attention agent for UltraVox.
-
-    Architecture: Whisper-large-v3-turbo encoder → stack_factor=8 projector → Llama-3.1-8B.
-    Audio is injected into the LLM embeddings at ``<|audio|>`` placeholder positions.
-    The pipeline preprocessor handles tokenization and audio feature extraction.
-
-    Extra config fields
-    -------------------
-    hf_model_name : str
-        Default: ``"fixie-ai/ultravox-v0_6-llama-3_1-8b"``.
-    repetition_penalty : float
-        Default: ``1.0``.
-    no_repeat_ngram_size : int
-        Default: ``0``.
-    """
-
-    BOW_PREFIX = " "
-    # Whisper-large-v3-turbo: 50 frames/s; stack_factor=8 → 50/8 tokens/s
-    # stride = 16000 / (50/8) = 2560 samples per audio token
-    AUDIO_TOKEN_STRIDE = 2560
-
-    def __init__(self, config: SimpleNamespace):
-        super().__init__(config)
-        self.bow_prefix = self.BOW_PREFIX
-        text_history_cls = class_load(self.text_history_config.type)
-        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
-        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
-        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
-        self.temperature = getattr(self.config, "temperature", 1.0)
-        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
-
-    @classmethod
-    def load_model(cls, config: SimpleNamespace) -> None:
-        model_name = getattr(
-            config,
-            "hf_model_name",
-            getattr(config, "model_path", "fixie-ai/ultravox-v0_6-llama-3_1-8b"),
-        )
-        attn_impl = getattr(config, "attn_implementation", "eager")
-
-        cls.pipe = transformers.pipeline(
-            model=model_name,
-            trust_remote_code=True,
-            torch_dtype=torch.bfloat16,
-            device_map="auto",
-            model_kwargs={"attn_implementation": attn_impl},
-        )
-        cls.model = cls.pipe.model
-        cls.model.eval()
-
-    def build_prompt(self) -> str:
-        return (
-            TEMPLATED_SPEECH_PROMPT
-            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
-            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
-        )
-
-    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        prefix = self.build_raw_text_prefix()
-        turns = [
-            {
-                "role": "system",
-                "content": self.build_prompt(),
-            },
-        ]
-        # The pipeline preprocessor tokenizes the conversation, extracts audio
-        # features, and returns audio_token_start_idx + audio_token_len alongside
-        # the standard input_ids/attention_mask/audio_values tensors.
-        inputs = self.pipe.preprocess(
-            {"audio": waveform, "turns": turns, "sampling_rate": SAMPLE_RATE}
-        )
-
-        if prefix:
-            prefix_ids = self.pipe.tokenizer(
-                prefix,
-                return_tensors="pt",
-                add_special_tokens=False,
-            ).input_ids
-            inputs["input_ids"] = torch.cat([inputs["input_ids"], prefix_ids], dim=1)
-            inputs["attention_mask"] = torch.cat(
-                [inputs["attention_mask"], torch.ones_like(prefix_ids)], dim=1
-            )
-
-        return {
-            k: v.to(self.device) if isinstance(v, torch.Tensor) else v
-            for k, v in inputs.items()
-        }
-
-    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
-        # Ultravox pipeline provides audio_token_start_idx and audio_token_len
-        # directly — no need to scan input_ids for a placeholder token.
-        # Called with the full inputs dict via _generate.
-        raise NotImplementedError("Use _find_audio_positions_from_inputs instead.")
-
-    def _find_audio_positions_from_inputs(self, inputs: dict) -> torch.Tensor:
-        start = inputs["audio_token_start_idx"][0].item()
-        length = inputs["audio_token_len"][0].item()
-        return torch.arange(start, start + length, device=self.device)
-
-    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
-        input_ids = inputs["input_ids"]
-        input_len = input_ids.shape[1]
-
-        audio_positions = self._find_audio_positions_from_inputs(inputs)
-        audio_len = audio_positions.shape[0]
-
-        output = self.model.generate(
-            **inputs,
-            max_new_tokens=self.max_new_tokens,
-            repetition_penalty=self.repetition_penalty,
-            no_repeat_ngram_size=self.no_repeat_ngram_size,
-            output_attentions=True,
-            return_dict_in_generate=True,
-            do_sample=False,
-            temperature=self.temperature,
-            eos_token_id=[
-                self.pipe.tokenizer.eos_token_id,
-                self.pipe.tokenizer.convert_tokens_to_ids("<|eot_id|>"),
-            ],
-        )
-
-        new_ids = output.sequences[:, input_len:]
-        new_tokens = [
-            self.pipe.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in new_ids[0]
-        ]
-
-        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        prefix_len = len(self.text_history) if self.text_history else 0
-        empty_attn = torch.zeros(0, audio_len, device=self.device)
-
-        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
-            if prefix_len > 0 else empty_attn
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
-        new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:]
-        ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-
-        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
-        cross_attn = self.normalize_attn(cross_attn)
-        return new_tokens, cross_attn
-
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/voxtral_doa.py b/simulstream/server/speech_processors/voxtral_doa.py
deleted file mode 100644
index 55ed67f..0000000
--- a/simulstream/server/speech_processors/voxtral_doa.py
+++ /dev/null
@@ -1,182 +0,0 @@
-# Copyright 2026 FBK
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-
-#     http://www.apache.org/licenses/LICENSE-2.0
-
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License
-
-import logging
-import base64
-import io
-from types import SimpleNamespace
-from typing import List, Tuple
-
-import numpy as np
-import soundfile as sf
-import torch
-
-from transformers import AutoProcessor, VoxtralForConditionalGeneration
-
-from simulstream.server.speech_processors import SAMPLE_RATE, class_load
-from simulstream.server.speech_processors.base_doa import (
-    DecoderOnlyAttention,
-    LANG_MAPPER,
-)
-
-from transformers import set_seed
-torch.manual_seed(42)
-set_seed(42)
-
-
-SUGGESTED_PROMPT = \
-    ("You are an expert multilingual speech translator. Your task is to accurately translate "
-     "the provided audio from {src_lang} into {tgt_lang}. Output ONLY the translated text. "
-     "Maintain the original tone, context, and speaker intent without adding any extra "
-     "conversational filler.")
-
-
-logger = logging.getLogger(__name__)
-
-
-class VoxtralDOA(DecoderOnlyAttention):
-    """
-    Decoder-Only Attention agent for ``mistralai/Voxtral-Mini-3B-2507`` and
-    ``mistralai/Voxtral-Small-24B-2507``.
-
-    Extra config fields
-    -------------------
-    hf_model_name : str
-        Default: ``"mistralai/Voxtral-Mini-3B-2507"``.
-    repetition_penalty : float
-        Default: ``1.0``.
-    no_repeat_ngram_size : int
-        Default: ``0``.
-    """
-
-    BOW_PREFIX = " "
-    # To be resolved from model config at load_model time
-    AUDIO_TOKEN_INDEX = None
-    # Voxtral encoder: 50 frames/s, stride 4 → 12.5 tokens/s
-    # stride = 16000 / 12.5 = 1280 samples per audio token
-    AUDIO_TOKEN_STRIDE = 1280
-
-    def __init__(self, config: SimpleNamespace):
-        super().__init__(config)
-        self.bow_prefix = self.BOW_PREFIX
-        text_history_cls = class_load(self.text_history_config.type)
-        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
-        self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
-        self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.0)
-        self.temperature = getattr(self.config, "temperature", 1.0)
-        self.no_repeat_ngram_size = getattr(self.config, "no_repeat_ngram_size", 0)
-
-    @classmethod
-    def load_model(cls, config: SimpleNamespace) -> None:
-        model_name = getattr(
-            config,
-            "hf_model_name",
-            getattr(config, "model_path", "mistralai/Voxtral-Mini-3B-2507"),
-        )
-        attn_impl = getattr(config, "attn_implementation", "eager")
-
-        cls.processor = AutoProcessor.from_pretrained(model_name)
-        cls.model = VoxtralForConditionalGeneration.from_pretrained(
-            model_name,
-            torch_dtype=torch.bfloat16,
-            device_map="auto",
-            attn_implementation=attn_impl,
-        )
-        cls.model.eval()
-        cls.AUDIO_TOKEN_INDEX = cls.model.config.audio_token_id
-
-    def build_prompt(self) -> str:
-        return (
-            SUGGESTED_PROMPT
-            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
-            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
-        )
-
-    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
-        audio_buffer = io.BytesIO()
-        sf.write(audio_buffer, waveform, SAMPLE_RATE, format="WAV")
-        audio_base64 = base64.b64encode(audio_buffer.getvalue()).decode("utf-8")
-
-        conversation = [
-            {
-                "role": "user",
-                "content": [
-                    {"type": "audio", "base64": audio_base64},
-                    {"type": "text", "text": self.build_prompt()},
-                ],
-            },
-        ]
-        inputs = self.processor.apply_chat_template(conversation)
-
-        prefix = self.build_raw_text_prefix()
-        if prefix:
-            prefix_ids = self.processor.tokenizer(
-                prefix,
-                return_tensors="pt",
-                add_special_tokens=False,
-            ).input_ids
-            inputs["input_ids"] = torch.cat([inputs["input_ids"], prefix_ids], dim=1)
-            inputs["attention_mask"] = torch.cat(
-                [inputs["attention_mask"], torch.ones_like(prefix_ids)], dim=1
-            )
-
-        return inputs.to(self.device, dtype=torch.bfloat16)
-
-    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
-        return (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
-
-    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
-        input_ids = inputs["input_ids"]
-        input_len = input_ids.shape[1]
-
-        audio_positions = self._find_audio_positions(input_ids)
-        audio_len = audio_positions.shape[0]
-
-        output = self.model.generate(
-            **inputs,
-            max_new_tokens=self.max_new_tokens,
-            repetition_penalty=self.repetition_penalty,
-            no_repeat_ngram_size=self.no_repeat_ngram_size,
-            output_attentions=True,
-            return_dict_in_generate=True,
-            do_sample=False,
-            temperature=self.temperature,
-        )
-
-        new_ids = output.sequences[:, input_len:]
-        new_tokens = [
-            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in new_ids[0]
-        ]
-
-        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
-        prefix_len = len(self.text_history) if self.text_history else 0
-        empty_attn = torch.zeros(0, audio_len, device=self.device)
-
-        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
-            if prefix_len > 0 else empty_attn
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
-        new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:]
-        ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-
-        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
-        cross_attn = self.normalize_attn(cross_attn)
-        return new_tokens, cross_attn
-
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
\ No newline at end of file

From c6ab7577bc07b8f27c9e62c4a0b6c8f8c905cc8c Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 28 May 2026 14:45:45 +0200
Subject: [PATCH 135/157] Revert voxtral detokenizer

---
 simulstream/metrics/detokenizers.py | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/simulstream/metrics/detokenizers.py b/simulstream/metrics/detokenizers.py
index 21e4b2d..72fbad9 100644
--- a/simulstream/metrics/detokenizers.py
+++ b/simulstream/metrics/detokenizers.py
@@ -21,25 +21,10 @@ def build_hf_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
 
     assert hasattr(config, "hf_model_name"), \
         "`hf_model_name` required in the eval config for `hf` detokenizer"
-    processor = AutoProcessor.from_pretrained(config.hf_model_name, trust_remote_code=True)
-    tokenizer = processor.tokenizer if hasattr(processor, "tokenizer") else processor
+    processor = AutoProcessor.from_pretrained(config.hf_model_name)
 
     def detokenize(input_tokens: List[str]) -> str:
-        return tokenizer.convert_tokens_to_string(input_tokens)
-
-    return detokenize
-
-def build_voxtral_detokenizer(config: SimpleNamespace) -> Callable[[List[str]], str]:
-    from transformers import AutoTokenizer
-
-    assert hasattr(config, "hf_model_name"), \
-        "`hf_model_name` required in the eval config for `voxtral` detokenizer"
-    tokenizer = AutoTokenizer.from_pretrained(config.hf_model_name)
-
-    def detokenize(input_tokens: List[str]) -> str:
-        text = "".join(input_tokens)
-        ids = tokenizer.encode(text, add_special_tokens=False)
-        return tokenizer.tokenizer.decode(ids)
+        return processor.tokenizer.convert_tokens_to_string(input_tokens)
 
     return detokenize
 
@@ -78,8 +63,7 @@ def detokenize(input_tokens: List[str]) -> str:
 _DETOKENIZER_BUILDER_MAP: Dict[str, Callable[[SimpleNamespace], Callable[[List[str]], str]]] = {
     "hf": build_hf_detokenizer,
     "canary": build_canary_detokenizer,
-    "simuleval": build_simuleval_detokenizer,
-    "voxtral": build_voxtral_detokenizer,
+    "simuleval": build_simuleval_detokenizer
 }
 
 

From 35e580978a1893db680755b107563ddb46176310 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 28 May 2026 14:52:59 +0200
Subject: [PATCH 136/157] Fix Phi4-Multimodal linting

---
 .../speech_processors/phi4multimodal_doa.py   |  9 +++--
 .../server/speech_processors/qwenomni_doa.py  | 33 +++++++++++++++++--
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 8930159..30650f1 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -34,10 +34,10 @@ class Phi4MultimodalDOA(DecoderOnlyAttention):
     """
 
     # Phi-4 special tokens
-    _USER_START  = "<|user|>"
+    _USER_START = "<|user|>"
     _AUDIO_TOKEN = "<|audio_1|>"
-    _END_TOKEN   = "<|end|>"
-    _ASST_START  = "<|assistant|>"
+    _END_TOKEN = "<|end|>"
+    _ASST_START = "<|assistant|>"
 
     BOW_PREFIX = " "
     ENCODER_SUBSAMPLING_FACTOR = 8
@@ -149,11 +149,10 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
 
-        cross_attn = torch.cat([prefix_rows, new_attn], dim=0) # (n_prefix + n_new, audio_len)
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)  # (n_prefix + n_new, audio_len)
         cross_attn = self.normalize_attn(cross_attn)
 
         return new_tokens, cross_attn
 
-
     def tokens_to_string(self, tokens: List[str]) -> str:
         return "".join(tokens)
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 6ede630..87a8c79 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -73,6 +73,7 @@ def __init__(self, config: SimpleNamespace):
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
+        """Load the Qwen3-Omni model and processor."""
         model_name = getattr(
             config,
             "hf_model_name",
@@ -91,6 +92,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
         cls.model.eval()
 
     def build_prompt(self) -> str:
+        """Build the translation instruction used alongside the audio input."""
         return (
             TEMPLATED_SPEECH_PROMPT
             .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
@@ -98,6 +100,7 @@ def build_prompt(self) -> str:
         )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+        """Build multimodal processor inputs from the rolling audio history."""
         prompt_text = self.build_prompt()
         prefix = self.build_raw_text_prefix()
 
@@ -136,6 +139,7 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         return inputs.to(self.device).to(self.model.dtype)
 
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        """Return token positions corresponding to the encoded audio span."""
         audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
         if audio_positions.numel() > 0:
             return audio_positions
@@ -146,12 +150,31 @@ def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
 
     def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
-        input_ids = inputs["input_ids"]
+        """
+        Run greedy generation and build the proxy cross-attention matrix.
+
+        Qwen3-Omni returns the thinker self-attention scores for each decode
+        step and layer. H is the dimension of the attention heads.
+
+        output.attentions[0][layer] -> (1, H, input_len, input_len)  # prefill
+        output.attentions[i][layer] -> (1, H, 1, input_len+i)        # new token i
+
+        Returns
+        -------
+        List[str]
+            A list of the newly generated tokens (n_new).
+        torch.Tensor
+            Proxy cross-attention scores extracted from the self-attention scores
+            (prefix + n_new, audio_len).
+        """
+        input_ids = inputs["input_ids"]  # (1, input_len)
         input_len = input_ids.shape[1]
 
+        # Locate audio positions.
         audio_positions = self._find_audio_positions(input_ids)
         audio_len = audio_positions.shape[0]
 
+        # Generate.
         output = self.model.generate(
             **inputs,
             use_audio_in_video=True,
@@ -168,18 +191,23 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         if isinstance(output, tuple):
             output = output[0]
 
+        # Decode newly generated tokens only.
         new_ids = output.sequences[:, input_len:]
         new_tokens = [
             self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
             for token_id in new_ids[0]
         ]
 
+        # Build proxy cross-attention for the hypothesis (prefix + new_tokens).
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
         empty_attn = torch.zeros(0, audio_len, device=self.device)
 
+        # Prefix rows come from the prefill pass.
         prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
             if prefix_len > 0 else empty_attn
+        # The prefill pass predicts the first generated token, so its last prompt row
+        # is used as the first generated token's proxy audio-attention.
         first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
         new_rows = [
             self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
@@ -193,4 +221,5 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         return new_tokens, cross_attn
 
     def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
\ No newline at end of file
+        """Convert decoded tokens to the emitted text string."""
+        return "".join(tokens)

From d908dd87dc477de0fc6efa616ab05af4c6cee296 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 28 May 2026 14:54:26 +0200
Subject: [PATCH 137/157] Fix Qwen3-Omni linting and remove useless comments

---
 simulstream/server/speech_processors/qwenomni_doa.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 87a8c79..aabce94 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -73,7 +73,6 @@ def __init__(self, config: SimpleNamespace):
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
-        """Load the Qwen3-Omni model and processor."""
         model_name = getattr(
             config,
             "hf_model_name",
@@ -92,7 +91,6 @@ def load_model(cls, config: SimpleNamespace) -> None:
         cls.model.eval()
 
     def build_prompt(self) -> str:
-        """Build the translation instruction used alongside the audio input."""
         return (
             TEMPLATED_SPEECH_PROMPT
             .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
@@ -185,7 +183,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             thinker_output_attentions=True,
             thinker_return_dict_in_generate=True,
             thinker_do_sample=False,
-            #thinker_eos_token_id=[151643, 151645],
             temperature=self.temperature,
         )
         if isinstance(output, tuple):

From 5fa2b2937a0c5e60a0444291b36995cfa8214aea Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 28 May 2026 14:55:50 +0200
Subject: [PATCH 138/157] Clean from useless comments

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 7 +++----
 simulstream/server/speech_processors/qwenomni_doa.py       | 1 -
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 30650f1..929fc6a 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -102,12 +102,11 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         input_ids = inputs["input_ids"]  # (1, input_len)
         input_len = input_ids.shape[1]
 
-        # Locate audio positions ──────────────────────────────────────────────────────────────────
+        # Locate audio positions.
         AUDIO_SPECIAL_TOKEN_ID = 200011  # _AUDIO_SPECIAL_TOKEN_ID in modeling_phi4mm.py
         audio_positions = (input_ids[0] == AUDIO_SPECIAL_TOKEN_ID).nonzero(as_tuple=True)[0]
         audio_len = audio_positions.shape[0]
 
-        # Generate ────────────────────────────────────────────────────────────────────────────────
         output = self.model.generate(
             **inputs,
             max_new_tokens=self.max_new_tokens,
@@ -118,14 +117,14 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
             do_sample=False,
         )
 
-        # Decode newly generated tokens only ──────────────────────────────────────────────────────
+        # Decode newly generated tokens only.
         new_ids = output.sequences[:, input_len:]  # (1, n_new)
         new_tokens = [
             self.processor.tokenizer.decode([t], skip_special_tokens=True)
             for t in new_ids[0]
         ]
 
-        # Build proxy cross-attention for the hypothesis (prefix + new_tokens) ────────────────────
+        # Build proxy cross-attention for the hypothesis (prefix + new_tokens).
         # Prefix rows from the prefill pass
         # output.attentions[0][layer]: (1, H, input_len, input_len)
         prefill_attn = self.mean_attn_over_heads_and_selected_layers(
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index aabce94..e0a285b 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -172,7 +172,6 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         audio_positions = self._find_audio_positions(input_ids)
         audio_len = audio_positions.shape[0]
 
-        # Generate.
         output = self.model.generate(
             **inputs,
             use_audio_in_video=True,

From e40fa38b51d5a494411a19aafd159e349cbaddce Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 28 May 2026 15:01:53 +0200
Subject: [PATCH 139/157] Fix uts

---
 uts/speech_processors/test_streamatt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uts/speech_processors/test_streamatt.py b/uts/speech_processors/test_streamatt.py
index 180c408..030536b 100644
--- a/uts/speech_processors/test_streamatt.py
+++ b/uts/speech_processors/test_streamatt.py
@@ -21,7 +21,7 @@
 class TestPunctuationTextHistory(unittest.TestCase):
     def setUp(self):
         self.config = SimpleNamespace()
-        self.punctuation_text_history = PunctuationTextHistory(self.config)
+        self.punctuation_text_history = PunctuationTextHistory(self.config, "")
 
     def test_punctuation_last(self):
         """ Test PunctuationTextHistory method when the history ends with strong punctuation. """

From 25e024b91a21a3b18081a1410bab18a384855118 Mon Sep 17 00:00:00 2001
From: sarapapi <57095209+sarapapi@users.noreply.github.com>
Date: Tue, 9 Jun 2026 13:06:57 +0200
Subject: [PATCH 140/157] Update
 simulstream/server/speech_processors/qwenomni_doa.py

Co-authored-by: Marco Gaido <mgaido@fbk.eu>
---
 simulstream/server/speech_processors/qwenomni_doa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index e0a285b..2cd3615 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -53,9 +53,9 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
 
     BOW_PREFIX = " "
     AUDIO_TOKEN_STRIDE = 640
-    AUDIO_TOKEN_INDEX = 151675   # <|audio_pad|>
+    AUDIO_TOKEN_INDEX = 151675  # <|audio_pad|>
     AUDIO_START_TOKEN_ID = 151669  # <|audio_start|>
-    AUDIO_END_TOKEN_ID = 151670    # <|audio_end|>
+    AUDIO_END_TOKEN_ID = 151670  # <|audio_end|>
     SYSTEM_PROMPT = (
         "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of "
         "perceiving auditory and visual inputs, as well as generating text and speech."

From d2a556afea77ca0926fa33ad819fc5b1ab087a45 Mon Sep 17 00:00:00 2001
From: sarapapi <57095209+sarapapi@users.noreply.github.com>
Date: Tue, 9 Jun 2026 13:07:20 +0200
Subject: [PATCH 141/157] Update
 simulstream/server/speech_processors/phi4multimodal_doa.py

Co-authored-by: Marco Gaido <mgaido@fbk.eu>
---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 929fc6a..2494aa0 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -41,7 +41,7 @@ class Phi4MultimodalDOA(DecoderOnlyAttention):
 
     BOW_PREFIX = " "
     ENCODER_SUBSAMPLING_FACTOR = 8
-    HOP_LENGTH = 160    # 10ms at 16kHz
+    HOP_LENGTH = 160  # 10ms at 16kHz
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)

From 426af02b8e0d160942afbda1d1376d6da0ad73b6 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Tue, 9 Jun 2026 18:34:16 +0200
Subject: [PATCH 142/157] Partially address comments

---
 .../server/speech_processors/base_doa.py      | 104 ++++++++----------
 .../speech_processors/phi4multimodal_doa.py   |  53 +++++----
 .../server/speech_processors/qwenomni_doa.py  |  45 ++++----
 3 files changed, 94 insertions(+), 108 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index e541000..f5adc0e 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -39,51 +39,32 @@
 
 class DecoderOnlyAttention(BaseStreamAtt):
     """
-    Generic Decoder-only Attention-based (DOA) policy for SpeechLLMs.
+    Generic Decoder-only Attention-based policy for SpeechLLMs.
 
     The class handles:
-    - Raw-waveform history accumulation.
-    - Greedy generation with ``output_attentions=True``.
-    - Building the proxy cross-attention matrix from self-attention weights.
-    - Applying StreamAtt-based policy on the proxy cross-attention matrix.
-
-    Subclasses must implement the five abstract methods listed below.
-
-    Parameters
-    ----------
-    config : SimpleNamespace
-        All fields from :class:`BaseStreamAtt`, plus:
-        cross_attn_layer : int
-            Layer from which to extract attention scores. Default: ``0``.
-        cross_attn_head : int | None
-            Attention head to use. If ``None``, attention scores are averaged
-            over all heads. If set together with
-            ``average_attn_over_layers=True``, the selected head is averaged
-            across layers. Default: ``None``.
-        average_attn_over_layers : bool
-            Whether to average attention over all decoder layers instead of
-            using the single layer selected by ``attn_layer``.
-            Default: ``False``.
-        audio_history_max_duration : int
-            Maximum raw waveform length to keep in the rolling history.
-            Default: ``180`` (seconds).
-        device : torch.device
-            Device to use for model's loading and execution.
-        max_new_tokens : int
-            Maximum tokens to generate per chunk.  Default: ``32``.
-
-    Supported attention-selection modes
-    -----------------------------------
-    - ``attn_head=None`` and ``average_attn_over_layers=True``:
-      average across layers and heads.
-    - ``attn_head=None`` and ``average_attn_over_layers=False``:
-      average across heads within ``attn_layer``.
-    - ``attn_head=<int>`` and ``average_attn_over_layers=True``:
-      average across layers within the selected head.
-
-    An additional mode is also supported for completeness:
-    - ``attn_head=<int>`` and ``average_attn_over_layers=False``:
-      use the selected head within ``attn_layer``.
+       - Raw-waveform history accumulation.
+       - Greedy generation with ``output_attentions=True``.
+       - Building the proxy cross-attention matrix from self-attention weights.
+        - Applying the StreamAtt-based policy on the proxy cross-attention matrix.
+
+    The derived class should implement the following methods:
+        - **load_model**: Loads the model and processor.
+        - **build_prompt**: Builds the text prompt to use with audio inputs.
+        - **build_processor_inputs**: Builds model inputs from the rolling audio history.
+        - **_generate**: Generates tokens and proxy cross-attention scores.
+        - **tokens_to_string**: Converts decoded tokens to a plain string.
+
+    Args:
+       config (SimpleNamespace): Configuration object. The following additional attributes are
+           expected:
+           - **attn_layer (int)**: Layer from which to extract attention scores. Defaults to 0.
+           - **attn_head (int)**: Attention head to use. If not set, attention scores are averaged
+             over all heads.
+           - **average_attn_over_layers (bool)**: Whether to average the selected attention view
+             over all decoder layers. Defaults to True.
+           - **audio_history_max_duration (int)**: Maximum raw waveform length to keep in the
+             rolling history, in seconds. Defaults to 180.
+           - **max_new_tokens (int)**: Maximum tokens to generate per chunk. Defaults to 32.
     """
 
     def __init__(self, config: SimpleNamespace):
@@ -122,16 +103,14 @@ def build_prompt(self) -> str:
     @abstractmethod
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         """
-        Given the *entire* rolling waveform history (float32, 16 kHz), return
-        a ``dict`` of ``torch.Tensor`` inputs ready to be passed to
-        ``self.model.generate(**inputs, …)``.
+        Build processor inputs from the entire rolling waveform history (float32, 16 kHz).
 
-        The tensors must already be on ``self.device``.
+        The returned tensors must already be on ``self.device``.
         """
         ...
 
     @abstractmethod
-    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+    def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
         """
         Generate tokens from the given inputs together with the self-attention scores.
 
@@ -158,30 +137,39 @@ def build_raw_text_prefix(self) -> str:
         return "".join(self.text_history) if self.text_history else ""
 
     def _select_attn_from_layer(self, layer_attn: torch.Tensor) -> torch.Tensor:
+        # Generation runs one stream at a time, so remove the singleton batch dimension.
+        layer_attn = layer_attn.squeeze(0)
         if self.cross_attn_head is None:
             # Default behavior: average over all heads for this layer.
-            return layer_attn[0].mean(dim=0)
+            return layer_attn.mean(dim=0)
 
-        num_heads = layer_attn.shape[1]
+        num_heads = layer_attn.shape[0]
         if self.cross_attn_head < 0 or self.cross_attn_head >= num_heads:
             raise ValueError(
                 f"Invalid attn_head={self.cross_attn_head}. Layer has {num_heads} heads."
             )
-        return layer_attn[0, self.cross_attn_head]
+        return layer_attn[self.cross_attn_head]
 
-    def mean_attn_over_heads_and_selected_layers(self, step_attn) -> torch.Tensor:
+    def average_attn(self, attn) -> torch.Tensor:
+        """
+        Average or select attentions according to ``attn_layer``, ``attn_head``, and
+        ``average_attn_over_layers``.
+
+        If ``attn_head`` is not set, attention is averaged over heads. If
+        ``average_attn_over_layers`` is set, the selected per-layer attention view is also averaged
+        across layers; otherwise only ``attn_layer`` is used.
+        """
         if self.average_attn_over_layers:
             # Average the per-layer attention view selected by _select_attn_from_layer.
             return torch.stack(
-                [self._select_attn_from_layer(layer_attn) for layer_attn in step_attn],
+                [self._select_attn_from_layer(layer_attn) for layer_attn in attn],
                 dim=0,
             ).mean(dim=0)
-        return self._select_attn_from_layer(step_attn[self.cross_attn_layer])
+        return self._select_attn_from_layer(attn[self.cross_attn_layer])
 
-    def _preprocess(self, waveform: np.float32) -> dict:
+    def _preprocess(self, waveform: np.float32) -> np.ndarray:
         """
-        Append *waveform* to ``self.audio_history``, enforce the maximum length,
-        and delegate to :meth:`build_processor_inputs`.
+        Append *waveform* to ``self.audio_history`` and enforce the maximum length.
         """
         if self.audio_history is None:
             self.audio_history = waveform
@@ -192,4 +180,4 @@ def _preprocess(self, waveform: np.float32) -> dict:
             logger.warning("Audio history exceeded %d samples; trimming.", self.audio_max_len)
             self.audio_history = self.audio_history[-self.audio_max_len:]
 
-        return self.build_processor_inputs(self.audio_history)
+        return self.audio_history
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 2494aa0..4164ff2 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -12,20 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-import torch
-import numpy as np
-
 from types import SimpleNamespace
 from typing import List, Tuple
 
+import numpy as np
+import torch
 from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
 
 from simulstream.server.speech_processors import SAMPLE_RATE, class_load
-from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, LANG_MAPPER
-
-from transformers import set_seed
-torch.manual_seed(42)
-set_seed(42)
+from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention
 
 
 class Phi4MultimodalDOA(DecoderOnlyAttention):
@@ -42,6 +37,7 @@ class Phi4MultimodalDOA(DecoderOnlyAttention):
     BOW_PREFIX = " "
     ENCODER_SUBSAMPLING_FACTOR = 8
     HOP_LENGTH = 160  # 10ms at 16kHz
+    AUDIO_SPECIAL_TOKEN_ID = 200011  # _AUDIO_SPECIAL_TOKEN_ID in modeling_phi4mm.py
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
@@ -51,7 +47,11 @@ def __init__(self, config: SimpleNamespace):
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
-        model_path = "microsoft/Phi-4-multimodal-instruct"
+        model_path = getattr(
+            config,
+            "hf_model_name",
+            getattr(config, "model_path", "microsoft/Phi-4-multimodal-instruct"),
+        )
 
         cls.processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
         cls.model = AutoModelForCausalLM.from_pretrained(
@@ -81,30 +81,31 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             return_tensors="pt",
         ).to(self.device)
 
-    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+    def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
         """
         Run greedy generation and build the proxy cross-attention matrix.
 
         ``output.attentions`` (use_cache=True) contains the self-attention scores,
         for each step and layer. H is the dimension of the attention heads.
-        ───────────────────────────────────────────────────────────────────────────────────────────
-        output.attentions[0][layer]  → (1, H, input_len, input_len)  ← prefill
-        output.attentions[i][layer]  → (1, H, 1, input_len+i)        ← new token i
-
-        Returns
-        -------
-        List[str]
-            A list of the newly generated tokens (n_new).
-        torch.Tensor
-            Proxy cross-attention scores extracted from the self-attention scores,
-            averaged over heads at ``self.cross_attn_layer`` (prefix + n_new, audio_len).
+
+        output.attentions[0][layer] -> (1, H, input_len, input_len)  # prefill
+        output.attentions[i][layer] -> (1, H, 1, input_len+i)        # new token i
+
+        Args:
+            waveform (np.ndarray): Rolling audio history.
+
+        Returns:
+            Tuple[List[str], torch.Tensor]:
+                List[str]: A list of the newly generated tokens.
+                torch.Tensor: Proxy cross-attention scores extracted from the self-attention scores
+                with dimension (prefix + generated_tokens, audio_len).
         """
+        inputs = self.build_processor_inputs(waveform)
         input_ids = inputs["input_ids"]  # (1, input_len)
         input_len = input_ids.shape[1]
 
         # Locate audio positions.
-        AUDIO_SPECIAL_TOKEN_ID = 200011  # _AUDIO_SPECIAL_TOKEN_ID in modeling_phi4mm.py
-        audio_positions = (input_ids[0] == AUDIO_SPECIAL_TOKEN_ID).nonzero(as_tuple=True)[0]
+        audio_positions = (input_ids[0] == self.AUDIO_SPECIAL_TOKEN_ID).nonzero(as_tuple=True)[0]
         audio_len = audio_positions.shape[0]
 
         output = self.model.generate(
@@ -127,9 +128,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         # Build proxy cross-attention for the hypothesis (prefix + new_tokens).
         # Prefix rows from the prefill pass
         # output.attentions[0][layer]: (1, H, input_len, input_len)
-        prefill_attn = self.mean_attn_over_heads_and_selected_layers(
-            output.attentions[0]
-        )  # (input_len, input_len)
+        prefill_attn = self.average_attn(output.attentions[0])  # (input_len, input_len)
         prefix_len = len(self.text_history) if self.text_history else 0
         if prefix_len > 0:
             prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
@@ -140,7 +139,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         first_new_row = prefill_attn[-1:, audio_positions] if len(new_tokens) > 0 else \
             torch.zeros(0, max(audio_len, 1), device=self.device)
         new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn)
+            self.average_attn(step_attn)
             .squeeze(0)[audio_positions]  # (audio_len,)
             for step_attn in output.attentions[1:]
         ]
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 2cd3615..bfa0d98 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -29,10 +29,6 @@
     TEMPLATED_SPEECH_PROMPT,
 )
 
-from transformers import set_seed
-torch.manual_seed(42)
-set_seed(42)
-
 
 logger = logging.getLogger(__name__)
 
@@ -41,14 +37,13 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
     """
     Decoder-Only Attention agent for Qwen3-Omni.
 
-    Extra config fields
-    -------------------
-    repetition_penalty : float
-        Repetition penalty for text generation. Default: ``1.05``.
-    temperature : float
-        Temperature for text generation. Default: ``1.0``.
-    no_repeat_ngram_size : int
-        N-gram blocking size for text generation. Default: ``5``.
+    Args:
+       config (SimpleNamespace): Configuration object. The following additional attributes are
+           expected:
+           - **repetition_penalty (float)**: Repetition penalty for text generation.
+           Default: ``1.05``.
+           - **temperature (float)**: Temperature parameter. Default: ``1.0``.
+           - **no_repeat_ngram_size (int)**: Ngram size for text generation. Default: ``5``.
     """
 
     BOW_PREFIX = " "
@@ -73,6 +68,7 @@ def __init__(self, config: SimpleNamespace):
 
     @classmethod
     def load_model(cls, config: SimpleNamespace) -> None:
+        """Load the Qwen3-Omni model and processor."""
         model_name = getattr(
             config,
             "hf_model_name",
@@ -80,6 +76,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
         )
         attn_impl = getattr(config, "attn_implementation", "eager")
 
+        cls.processor = Qwen3OmniMoeProcessor.from_pretrained(model_name)
         cls.model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(
             model_name,
             torch_dtype="auto",
@@ -87,10 +84,10 @@ def load_model(cls, config: SimpleNamespace) -> None:
             attn_implementation=attn_impl,
             enable_audio_output=False,
         )
-        cls.processor = Qwen3OmniMoeProcessor.from_pretrained(model_name)
         cls.model.eval()
 
     def build_prompt(self) -> str:
+        """Build the translation instruction used alongside the audio input."""
         return (
             TEMPLATED_SPEECH_PROMPT
             .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
@@ -147,7 +144,7 @@ def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         end_pos = end_pos[end_pos > start_pos][0]
         return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
 
-    def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
+    def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
         """
         Run greedy generation and build the proxy cross-attention matrix.
 
@@ -157,14 +154,16 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         output.attentions[0][layer] -> (1, H, input_len, input_len)  # prefill
         output.attentions[i][layer] -> (1, H, 1, input_len+i)        # new token i
 
-        Returns
-        -------
-        List[str]
-            A list of the newly generated tokens (n_new).
-        torch.Tensor
-            Proxy cross-attention scores extracted from the self-attention scores
-            (prefix + n_new, audio_len).
+        Args:
+            waveform (np.ndarray): Rolling audio history.
+
+        Returns:
+            Tuple[List[str], torch.Tensor]:
+                List[str]: A list of the newly generated tokens.
+                torch.Tensor: Proxy cross-attention scores extracted from the self-attention scores
+                with dimension (prefix + generated_tokens, audio_len).
         """
+        inputs = self.build_processor_inputs(waveform)
         input_ids = inputs["input_ids"]  # (1, input_len)
         input_len = input_ids.shape[1]
 
@@ -195,7 +194,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         ]
 
         # Build proxy cross-attention for the hypothesis (prefix + new_tokens).
-        prefill_attn = self.mean_attn_over_heads_and_selected_layers(output.attentions[0])
+        prefill_attn = self.average_attn(output.attentions[0])
         prefix_len = len(self.text_history) if self.text_history else 0
         empty_attn = torch.zeros(0, audio_len, device=self.device)
 
@@ -206,7 +205,7 @@ def _generate(self, inputs: dict) -> Tuple[List[str], torch.Tensor]:
         # is used as the first generated token's proxy audio-attention.
         first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
         new_rows = [
-            self.mean_attn_over_heads_and_selected_layers(step_attn).squeeze(0)[audio_positions]
+            self.average_attn(step_attn).squeeze(0)[audio_positions]
             for step_attn in output.attentions[1:]
         ]
         subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn

From cd5ecc425147d221a787cd1e9f12cb8196bf2ff6 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 11 Jun 2026 10:28:48 +0200
Subject: [PATCH 143/157] Address comment about pycountry

---
 pyproject.toml                                             | 3 ++-
 simulstream/server/speech_processors/base_doa.py           | 6 +++++-
 simulstream/server/speech_processors/phi4multimodal_doa.py | 4 ++--
 simulstream/server/speech_processors/qwenomni_doa.py       | 6 +++---
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 45a6581..e4912cb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,7 +14,8 @@ dependencies = [
     "pyyaml>6.0",
     "websockets",
     "torch",
-    "librosa"
+    "librosa",
+    "pycountry"
 ]
 dynamic = ["version"]
 
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index f5adc0e..ecd755d 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -18,6 +18,7 @@
 from typing import List, Tuple
 
 import numpy as np
+import pycountry
 import torch
 
 from simulstream.server.speech_processors import SAMPLE_RATE
@@ -34,7 +35,10 @@
      "translation, without any additional explanations or commentary. Please translate the "
      "provided {src_lang} speech into {tgt_lang}:")
 
-LANG_MAPPER = {"en": "English", "it": "Italian", "de": "German", "zh": "Chinese (simplified)"}
+def get_language_name(code: str) -> str:
+    """Return the language name for an ISO 639-1 code, falling back to the code itself."""
+    lang = pycountry.languages.get(alpha_2=code)
+    return lang.name if lang is not None else code
 
 
 class DecoderOnlyAttention(BaseStreamAtt):
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 4164ff2..1022222 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -20,7 +20,7 @@
 from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
 
 from simulstream.server.speech_processors import SAMPLE_RATE, class_load
-from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention
+from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, get_language_name
 
 
 class Phi4MultimodalDOA(DecoderOnlyAttention):
@@ -65,7 +65,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
         cls.generation_config = GenerationConfig.from_pretrained(model_path)
 
     def build_prompt(self) -> str:
-        filled_prompt = f"Translate the audio to {LANG_MAPPER[self.tgt_lang]}."
+        filled_prompt = f"Translate the audio to {get_language_name(self.tgt_lang)}."
         raw_prefix = self.build_raw_text_prefix()
         prompt = (
             f"{self._USER_START}{self._AUDIO_TOKEN}"
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index bfa0d98..1369b63 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -25,7 +25,7 @@
 from simulstream.server.speech_processors import SAMPLE_RATE, class_load
 from simulstream.server.speech_processors.base_doa import (
     DecoderOnlyAttention,
-    LANG_MAPPER,
+    get_language_name,
     TEMPLATED_SPEECH_PROMPT,
 )
 
@@ -90,8 +90,8 @@ def build_prompt(self) -> str:
         """Build the translation instruction used alongside the audio input."""
         return (
             TEMPLATED_SPEECH_PROMPT
-            .replace("{src_lang}", LANG_MAPPER.get(self.src_lang, self.src_lang))
-            .replace("{tgt_lang}", LANG_MAPPER.get(self.tgt_lang, self.tgt_lang))
+            .replace("{src_lang}", get_language_name(self.src_lang))
+            .replace("{tgt_lang}", get_language_name(self.tgt_lang))
         )
 
     def build_processor_inputs(self, waveform: np.ndarray) -> dict:

From 26e06ae629889083639d8bd1cd4ec06dac1ca6df Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 11 Jun 2026 10:33:07 +0200
Subject: [PATCH 144/157] Fix linting

---
 simulstream/server/speech_processors/base_doa.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index ecd755d..b7e0368 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -35,6 +35,7 @@
      "translation, without any additional explanations or commentary. Please translate the "
      "provided {src_lang} speech into {tgt_lang}:")
 
+
 def get_language_name(code: str) -> str:
     """Return the language name for an ISO 639-1 code, falling back to the code itself."""
     lang = pycountry.languages.get(alpha_2=code)

From 76ba01350703fd58adf9dcb84fdbd496d49f2a26 Mon Sep 17 00:00:00 2001
From: Marco Gaido <mgaido@fbk.eu>
Date: Fri, 12 Jun 2026 17:17:06 +0200
Subject: [PATCH 145/157] refactor code to avoid duplicated code

---
 .../server/speech_processors/base_doa.py      | 66 +++++++++++++---
 .../speech_processors/phi4multimodal_doa.py   | 75 +++----------------
 .../server/speech_processors/qwenomni_doa.py  | 72 +++---------------
 3 files changed, 77 insertions(+), 136 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index b7e0368..4aed5ee 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -15,7 +15,7 @@
 import logging
 from abc import abstractmethod
 from types import SimpleNamespace
-from typing import List, Tuple
+from typing import List, Tuple, Any, Dict
 
 import numpy as np
 import pycountry
@@ -50,14 +50,15 @@ class DecoderOnlyAttention(BaseStreamAtt):
        - Raw-waveform history accumulation.
        - Greedy generation with ``output_attentions=True``.
        - Building the proxy cross-attention matrix from self-attention weights.
-        - Applying the StreamAtt-based policy on the proxy cross-attention matrix.
+       - Applying the StreamAtt-based policy on the proxy cross-attention matrix.
 
     The derived class should implement the following methods:
         - **load_model**: Loads the model and processor.
         - **build_prompt**: Builds the text prompt to use with audio inputs.
         - **build_processor_inputs**: Builds model inputs from the rolling audio history.
-        - **_generate**: Generates tokens and proxy cross-attention scores.
+        - **_do_generate**: Returns newly-generated tokens and self-attention scores.
         - **tokens_to_string**: Converts decoded tokens to a plain string.
+        - **_find_audio_positions**: Returns the indices of audio tokens.
 
     Args:
        config (SimpleNamespace): Configuration object. The following additional attributes are
@@ -106,15 +107,30 @@ def build_prompt(self) -> str:
         ...
 
     @abstractmethod
-    def build_processor_inputs(self, waveform: np.ndarray) -> dict:
+    def build_processor_inputs(self, waveform: np.ndarray) -> Any:
         """
         Build processor inputs from the entire rolling waveform history (float32, 16 kHz).
+        """
+        ...
 
-        The returned tensors must already be on ``self.device``.
+    @abstractmethod
+    def _do_generate(self, inputs: Dict[str, Any]) -> Tuple[List[str], List[torch.Tensor]]:
+        """
+        Runs the actual generation from the underlying model and returns the generated tokens and
+        the corresponding self-attention scores.
         """
         ...
 
     @abstractmethod
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        """Return token positions corresponding to the encoded audio span."""
+        ...
+
+    @abstractmethod
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert a list of decoded tokens to a plain output string."""
+        ...
+
     def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
         """
         Generate tokens from the given inputs together with the self-attention scores.
@@ -125,12 +141,42 @@ def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
                 torch.Tensor: Self-attention scores between speech and text with dimension
                 (token_len, audio_len).
         """
-        ...
+        inputs = self.build_processor_inputs(waveform).to(self.device)
+        input_ids = inputs["input_ids"]  # (1, input_len)
+        input_len = input_ids.shape[1]
+
+        audio_positions = self._find_audio_positions(input_ids)
+        audio_len = audio_positions.shape[0]
+
+        # Run the actual generate on the underlying model
+        new_tokens, attentions = self._do_generate(inputs)
+
+        # Build proxy cross-attention for the hypothesis (prefix + new_tokens)
+        prefill_attn = self.average_attn(attentions[0])
+        prefix_len = len(self.text_history) if self.text_history else 0
+        if prefix_len > 0:
+            # Prefix rows come from the prefill pass
+            prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
+        else:
+            prefix_rows = torch.zeros(0, audio_len, device=self.device)
+
+        if new_tokens:
+            # The prefill pass predicts the first generated token, so its last row corresponds to
+            # the first generated token's proxy audio-attention
+            first_new_row = prefill_attn[-1:, audio_positions]
+            # Other tokens' attention is present in each generation step
+            new_rows = [
+                self.average_attn(step_attn).squeeze(0)[audio_positions]
+                for step_attn in attentions[1:]
+            ]
+            subsequent_new_attn = torch.stack(new_rows, dim=0)
+            new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
+        else:
+            new_attn = torch.zeros(0, audio_len, device=self.device)
 
-    @abstractmethod
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        """Convert a list of decoded tokens to a plain output string."""
-        ...
+        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
+        cross_attn = self.normalize_attn(cross_attn)
+        return new_tokens, cross_attn
 
     def set_target_language(self, language: str) -> None:
         self.tgt_lang = language
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 1022222..fd788a0 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -13,7 +13,7 @@
 # limitations under the License
 
 from types import SimpleNamespace
-from typing import List, Tuple
+from typing import List, Any, Dict, Tuple
 
 import numpy as np
 import torch
@@ -79,35 +79,13 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             text=self.build_prompt(),
             audios=[(waveform, SAMPLE_RATE)],
             return_tensors="pt",
-        ).to(self.device)
-
-    def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
-        """
-        Run greedy generation and build the proxy cross-attention matrix.
-
-        ``output.attentions`` (use_cache=True) contains the self-attention scores,
-        for each step and layer. H is the dimension of the attention heads.
-
-        output.attentions[0][layer] -> (1, H, input_len, input_len)  # prefill
-        output.attentions[i][layer] -> (1, H, 1, input_len+i)        # new token i
-
-        Args:
-            waveform (np.ndarray): Rolling audio history.
-
-        Returns:
-            Tuple[List[str], torch.Tensor]:
-                List[str]: A list of the newly generated tokens.
-                torch.Tensor: Proxy cross-attention scores extracted from the self-attention scores
-                with dimension (prefix + generated_tokens, audio_len).
-        """
-        inputs = self.build_processor_inputs(waveform)
-        input_ids = inputs["input_ids"]  # (1, input_len)
-        input_len = input_ids.shape[1]
+        )
 
-        # Locate audio positions.
-        audio_positions = (input_ids[0] == self.AUDIO_SPECIAL_TOKEN_ID).nonzero(as_tuple=True)[0]
-        audio_len = audio_positions.shape[0]
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens)
 
+    def _do_generate(self, inputs: Dict[str, Any]) -> Tuple[List[str], List[torch.Tensor]]:
+        input_len = inputs["input_ids"].shape[1]
         output = self.model.generate(
             **inputs,
             max_new_tokens=self.max_new_tokens,
@@ -117,40 +95,9 @@ def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
             return_dict_in_generate=True,
             do_sample=False,
         )
+        new_tokens = self.processor.tokenizer.convert_ids_to_tokens(
+            output.sequences[0, input_len:], skip_special_tokens=True)
+        return new_tokens, output.attentions
 
-        # Decode newly generated tokens only.
-        new_ids = output.sequences[:, input_len:]  # (1, n_new)
-        new_tokens = [
-            self.processor.tokenizer.decode([t], skip_special_tokens=True)
-            for t in new_ids[0]
-        ]
-
-        # Build proxy cross-attention for the hypothesis (prefix + new_tokens).
-        # Prefix rows from the prefill pass
-        # output.attentions[0][layer]: (1, H, input_len, input_len)
-        prefill_attn = self.average_attn(output.attentions[0])  # (input_len, input_len)
-        prefix_len = len(self.text_history) if self.text_history else 0
-        if prefix_len > 0:
-            prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions]
-        else:
-            prefix_rows = torch.zeros(0, max(audio_len, 1), device=self.device)
-        # The prefill pass predicts the first generated token, so we use the last prompt row
-        # as its proxy audio-attention. Subsequent generated tokens come from later decode steps.
-        first_new_row = prefill_attn[-1:, audio_positions] if len(new_tokens) > 0 else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
-        new_rows = [
-            self.average_attn(step_attn)
-            .squeeze(0)[audio_positions]  # (audio_len,)
-            for step_attn in output.attentions[1:]
-        ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else \
-            torch.zeros(0, max(audio_len, 1), device=self.device)
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-
-        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)  # (n_prefix + n_new, audio_len)
-        cross_attn = self.normalize_attn(cross_attn)
-
-        return new_tokens, cross_attn
-
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
+    def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return (input_ids[0] == self.AUDIO_SPECIAL_TOKEN_ID).nonzero(as_tuple=True)[0]
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 1369b63..4f27333 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -14,11 +14,10 @@
 
 import logging
 from types import SimpleNamespace
-from typing import List, Tuple
+from typing import List, Dict, Any, Tuple
 
 import numpy as np
 import torch
-
 from qwen_omni_utils import process_mm_info
 from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor
 
@@ -133,6 +132,10 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         )
         return inputs.to(self.device).to(self.model.dtype)
 
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert decoded tokens to the emitted text string."""
+        return "".join(tokens)
+
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         """Return token positions corresponding to the encoded audio span."""
         audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]
@@ -144,33 +147,8 @@ def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         end_pos = end_pos[end_pos > start_pos][0]
         return torch.arange(start_pos + 1, end_pos, device=input_ids.device)
 
-    def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
-        """
-        Run greedy generation and build the proxy cross-attention matrix.
-
-        Qwen3-Omni returns the thinker self-attention scores for each decode
-        step and layer. H is the dimension of the attention heads.
-
-        output.attentions[0][layer] -> (1, H, input_len, input_len)  # prefill
-        output.attentions[i][layer] -> (1, H, 1, input_len+i)        # new token i
-
-        Args:
-            waveform (np.ndarray): Rolling audio history.
-
-        Returns:
-            Tuple[List[str], torch.Tensor]:
-                List[str]: A list of the newly generated tokens.
-                torch.Tensor: Proxy cross-attention scores extracted from the self-attention scores
-                with dimension (prefix + generated_tokens, audio_len).
-        """
-        inputs = self.build_processor_inputs(waveform)
-        input_ids = inputs["input_ids"]  # (1, input_len)
-        input_len = input_ids.shape[1]
-
-        # Locate audio positions.
-        audio_positions = self._find_audio_positions(input_ids)
-        audio_len = audio_positions.shape[0]
-
+    def _do_generate(self, inputs: Dict[str, Any]) -> Tuple[List[str], List[torch.Tensor]]:
+        input_len = inputs["input_ids"].shape[1]
         output = self.model.generate(
             **inputs,
             use_audio_in_video=True,
@@ -185,36 +163,6 @@ def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
         )
         if isinstance(output, tuple):
             output = output[0]
-
-        # Decode newly generated tokens only.
-        new_ids = output.sequences[:, input_len:]
-        new_tokens = [
-            self.processor.tokenizer.decode([token_id], skip_special_tokens=True)
-            for token_id in new_ids[0]
-        ]
-
-        # Build proxy cross-attention for the hypothesis (prefix + new_tokens).
-        prefill_attn = self.average_attn(output.attentions[0])
-        prefix_len = len(self.text_history) if self.text_history else 0
-        empty_attn = torch.zeros(0, audio_len, device=self.device)
-
-        # Prefix rows come from the prefill pass.
-        prefix_rows = prefill_attn[input_len - prefix_len:, :][:, audio_positions] \
-            if prefix_len > 0 else empty_attn
-        # The prefill pass predicts the first generated token, so its last prompt row
-        # is used as the first generated token's proxy audio-attention.
-        first_new_row = prefill_attn[-1:, audio_positions] if new_tokens else empty_attn
-        new_rows = [
-            self.average_attn(step_attn).squeeze(0)[audio_positions]
-            for step_attn in output.attentions[1:]
-        ]
-        subsequent_new_attn = torch.stack(new_rows, dim=0) if new_rows else empty_attn
-        new_attn = torch.cat([first_new_row, subsequent_new_attn], dim=0)
-
-        cross_attn = torch.cat([prefix_rows, new_attn], dim=0)
-        cross_attn = self.normalize_attn(cross_attn)
-        return new_tokens, cross_attn
-
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        """Convert decoded tokens to the emitted text string."""
-        return "".join(tokens)
+        new_tokens = self.processor.tokenizer.convert_ids_to_tokens(
+            output.sequences[0, input_len:], skip_special_tokens=True)
+        return new_tokens, output.attentions

From dae38d83a98691c6173cbc7accfa52d2a57d8431 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 18 Jun 2026 12:22:19 +0200
Subject: [PATCH 146/157] Fix logging

---
 simulstream/server/speech_processors/base_doa.py     | 8 +++++++-
 simulstream/server/speech_processors/qwenomni_doa.py | 3 ---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 4aed5ee..17a53f8 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -24,6 +24,7 @@
 from simulstream.server.speech_processors import SAMPLE_RATE
 from simulstream.server.speech_processors.base_streamatt import BaseStreamAtt
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -39,7 +40,12 @@
 def get_language_name(code: str) -> str:
     """Return the language name for an ISO 639-1 code, falling back to the code itself."""
     lang = pycountry.languages.get(alpha_2=code)
-    return lang.name if lang is not None else code
+    if lang is not None:
+        return lang.name
+    else:
+        logger.warning(f"Language code '{code}' not found in the language list. Using language "
+                       f"code directly in the prompt, but this can lead to unexpected behavior.")
+        return code
 
 
 class DecoderOnlyAttention(BaseStreamAtt):
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 4f27333..4c63618 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -29,9 +29,6 @@
 )
 
 
-logger = logging.getLogger(__name__)
-
-
 class Qwen3OmniDOA(DecoderOnlyAttention):
     """
     Decoder-Only Attention agent for Qwen3-Omni.

From a7ac4d4b728f0044dc5f411691b4907c36e99596 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 18 Jun 2026 12:24:32 +0200
Subject: [PATCH 147/157] Partially revert the change

---
 simulstream/server/speech_processors/base_streamatt.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 249c76a..57e4663 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -73,10 +73,10 @@ class BaseStreamAtt(BaseSpeechProcessor):
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
         self.config = config
-        self.text_history_config = self.config.text_history
-        text_history_cls = class_load(self.text_history_config.type)
+        text_history_config = self.config.text_history
+        text_history_cls = class_load(text_history_config.type)
         self.bow_prefix = getattr(self.config, "bow_prefix", BOW_PREFIX)
-        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
+        self.text_history_method = text_history_cls(text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = getattr(self.config, "audio_subsampling_factor", 1)
         self.text_history_max_len = getattr(self.config, "text_history_max_len", 128)
         self.cross_attn_layer = getattr(self.config, "cross_attention_layer", 3)

From 1fad0d7e21b40a61747613dd043035460875a6b2 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 18 Jun 2026 12:29:42 +0200
Subject: [PATCH 148/157] Fix the description to go new line after 100 chars

---
 simulstream/server/speech_processors/base_doa.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 17a53f8..146a638 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -96,12 +96,11 @@ def audio_max_len(self) -> int:
     @abstractmethod
     def load_model(self, config: SimpleNamespace) -> None:
         """
-        Load the model and processor from *config* and assign them to
-        ``self.model`` and ``self.processor``.
+        Load the model and processor from *config* and assign them to ``self.model`` and
+        ``self.processor``.
 
-        The model **must** be loaded with ``output_attentions=True`` (or the
-        equivalent flag for the architecture) and
-        ``_attn_implementation="eager"``.
+        The model **must** be loaded with ``output_attentions=True`` (or the equivalent flag for
+        the architecture) and ``_attn_implementation="eager"``.
         """
         ...
 
@@ -213,8 +212,8 @@ def average_attn(self, attn) -> torch.Tensor:
         ``average_attn_over_layers``.
 
         If ``attn_head`` is not set, attention is averaged over heads. If
-        ``average_attn_over_layers`` is set, the selected per-layer attention view is also averaged
-        across layers; otherwise only ``attn_layer`` is used.
+        ``average_attn_over_layers`` is set, the selected per-layer attention view is also averaged across
+        layers; otherwise only ``attn_layer`` is used.
         """
         if self.average_attn_over_layers:
             # Average the per-layer attention view selected by _select_attn_from_layer.

From 909d36d8a6ad4105db527349af20655606fe383c Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 18 Jun 2026 14:23:57 +0200
Subject: [PATCH 149/157] Address comments

---
 config/phi4multimodal_doa_fixedwords.yaml            |  3 ++-
 config/phi4multimodal_doa_punctuation.yaml           |  3 ++-
 config/qwen3omni_doa_punctuation.yaml                |  9 ++++++++-
 simulstream/server/speech_processors/base_doa.py     | 11 ++---------
 .../server/speech_processors/phi4multimodal_doa.py   |  5 +----
 simulstream/server/speech_processors/qwenomni_doa.py | 12 ++----------
 6 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/config/phi4multimodal_doa_fixedwords.yaml b/config/phi4multimodal_doa_fixedwords.yaml
index 78ace66..c91b678 100644
--- a/config/phi4multimodal_doa_fixedwords.yaml
+++ b/config/phi4multimodal_doa_fixedwords.yaml
@@ -12,4 +12,5 @@ average_attn_over_layers: True
 detokenizer_type: "hf"
 hf_model_name: "microsoft/Phi-4-multimodal-instruct"
 word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
+max_new_tokens: 32
+bow_prefix: " "
\ No newline at end of file
diff --git a/config/phi4multimodal_doa_punctuation.yaml b/config/phi4multimodal_doa_punctuation.yaml
index 804035f..b54ed39 100644
--- a/config/phi4multimodal_doa_punctuation.yaml
+++ b/config/phi4multimodal_doa_punctuation.yaml
@@ -11,4 +11,5 @@ average_attn_over_layers: True
 detokenizer_type: "hf"
 hf_model_name: "microsoft/Phi-4-multimodal-instruct"
 word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
+max_new_tokens: 32
+bow_prefix: " "
\ No newline at end of file
diff --git a/config/qwen3omni_doa_punctuation.yaml b/config/qwen3omni_doa_punctuation.yaml
index 96e6f02..81a6b6e 100644
--- a/config/qwen3omni_doa_punctuation.yaml
+++ b/config/qwen3omni_doa_punctuation.yaml
@@ -11,4 +11,11 @@ cutoff_frame_num: __FRAME__
 detokenizer_type: "hf"
 hf_model_name: "Qwen/Qwen3-Omni-30B-A3B-Instruct"
 word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
\ No newline at end of file
+max_new_tokens: 32
+bow_prefix: " "
+prompt: "You are a professional {src_lang}-to-{tgt_lang} translator. Your goal is to accurately \
+convey the meaning and nuances of the original {src_lang} speech while adhering to {tgt_lang} \
+grammar, vocabulary, and cultural sensitivities. Use precise terminology and a tone appropriate \
+for academic or instructional materials. Produce only the {tgt_lang} translation, without any \
+additional explanations or commentary. Please translate the provided {src_lang} speech into \
+{tgt_lang}:"
\ No newline at end of file
diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 146a638..618459a 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -28,15 +28,6 @@
 logger = logging.getLogger(__name__)
 
 
-TEMPLATED_SPEECH_PROMPT = \
-    ("You are a professional {src_lang}-to-{tgt_lang} translator. Your goal is to accurately "
-     "convey the meaning and nuances of the original {src_lang} speech while adhering to "
-     "{tgt_lang} grammar, vocabulary, and cultural sensitivities. Use precise terminology and a "
-     "tone appropriate for academic or instructional materials. Produce only the {tgt_lang} "
-     "translation, without any additional explanations or commentary. Please translate the "
-     "provided {src_lang} speech into {tgt_lang}:")
-
-
 def get_language_name(code: str) -> str:
     """Return the language name for an ISO 639-1 code, falling back to the code itself."""
     lang = pycountry.languages.get(alpha_2=code)
@@ -87,6 +78,8 @@ def __init__(self, config: SimpleNamespace):
         self.audio_history_max_duration = getattr(self.config, "audio_history_max_duration", 180)
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.max_new_tokens = getattr(self.config, "max_new_tokens", 32)
+        self.prompt = getattr(self.config, "prompt", "Translate the audio to {tgt_lang}:")
+        logger.debug("Prompt:\n%s", self.prompt)
 
     @property
     def audio_max_len(self) -> int:
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index fd788a0..afe7857 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -34,15 +34,12 @@ class Phi4MultimodalDOA(DecoderOnlyAttention):
     _END_TOKEN = "<|end|>"
     _ASST_START = "<|assistant|>"
 
-    BOW_PREFIX = " "
     ENCODER_SUBSAMPLING_FACTOR = 8
     HOP_LENGTH = 160  # 10ms at 16kHz
     AUDIO_SPECIAL_TOKEN_ID = 200011  # _AUDIO_SPECIAL_TOKEN_ID in modeling_phi4mm.py
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
-        text_history_cls = class_load(self.text_history_config.type)
-        self.text_history_method = text_history_cls(self.text_history_config, self.BOW_PREFIX)
         self.audio_subsampling_factor = self.ENCODER_SUBSAMPLING_FACTOR * self.HOP_LENGTH
 
     @classmethod
@@ -65,7 +62,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
         cls.generation_config = GenerationConfig.from_pretrained(model_path)
 
     def build_prompt(self) -> str:
-        filled_prompt = f"Translate the audio to {get_language_name(self.tgt_lang)}."
+        filled_prompt = self.prompt.replace("{tgt_lang}", get_language_name(self.tgt_lang))
         raw_prefix = self.build_raw_text_prefix()
         prompt = (
             f"{self._USER_START}{self._AUDIO_TOKEN}"
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 4c63618..7571145 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -22,11 +22,7 @@
 from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor
 
 from simulstream.server.speech_processors import SAMPLE_RATE, class_load
-from simulstream.server.speech_processors.base_doa import (
-    DecoderOnlyAttention,
-    get_language_name,
-    TEMPLATED_SPEECH_PROMPT,
-)
+from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, get_language_name
 
 
 class Qwen3OmniDOA(DecoderOnlyAttention):
@@ -42,7 +38,6 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
            - **no_repeat_ngram_size (int)**: Ngram size for text generation. Default: ``5``.
     """
 
-    BOW_PREFIX = " "
     AUDIO_TOKEN_STRIDE = 640
     AUDIO_TOKEN_INDEX = 151675  # <|audio_pad|>
     AUDIO_START_TOKEN_ID = 151669  # <|audio_start|>
@@ -54,9 +49,6 @@ class Qwen3OmniDOA(DecoderOnlyAttention):
 
     def __init__(self, config: SimpleNamespace):
         super().__init__(config)
-        self.bow_prefix = self.BOW_PREFIX
-        text_history_cls = class_load(self.text_history_config.type)
-        self.text_history_method = text_history_cls(self.text_history_config, self.bow_prefix)
         self.audio_subsampling_factor = self.AUDIO_TOKEN_STRIDE
         self.repetition_penalty = getattr(self.config, "repetition_penalty", 1.05)
         self.temperature = getattr(self.config, "temperature", 1.0)
@@ -85,7 +77,7 @@ def load_model(cls, config: SimpleNamespace) -> None:
     def build_prompt(self) -> str:
         """Build the translation instruction used alongside the audio input."""
         return (
-            TEMPLATED_SPEECH_PROMPT
+            self.prompt
             .replace("{src_lang}", get_language_name(self.src_lang))
             .replace("{tgt_lang}", get_language_name(self.tgt_lang))
         )

From 987637b829e58c29292c8935f3a717b7e33ed79a Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 18 Jun 2026 14:27:24 +0200
Subject: [PATCH 150/157] Fix Lint

---
 simulstream/server/speech_processors/base_doa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 618459a..52cacba 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -205,8 +205,8 @@ def average_attn(self, attn) -> torch.Tensor:
         ``average_attn_over_layers``.
 
         If ``attn_head`` is not set, attention is averaged over heads. If
-        ``average_attn_over_layers`` is set, the selected per-layer attention view is also averaged across
-        layers; otherwise only ``attn_layer`` is used.
+        ``average_attn_over_layers`` is set, the selected per-layer attention view is also
+        averaged across layers; otherwise only ``attn_layer`` is used.
         """
         if self.average_attn_over_layers:
             # Average the per-layer attention view selected by _select_attn_from_layer.

From 39d6d7b7061ad8b35bcbeb46f54418b2b1e7608c Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Thu, 18 Jun 2026 14:28:07 +0200
Subject: [PATCH 151/157] Remove unused import

---
 simulstream/server/speech_processors/phi4multimodal_doa.py | 2 +-
 simulstream/server/speech_processors/qwenomni_doa.py       | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index afe7857..521545c 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -19,7 +19,7 @@
 import torch
 from transformers import AutoModelForCausalLM, AutoProcessor, GenerationConfig
 
-from simulstream.server.speech_processors import SAMPLE_RATE, class_load
+from simulstream.server.speech_processors import SAMPLE_RATE
 from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, get_language_name
 
 
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 7571145..15a9c12 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-import logging
 from types import SimpleNamespace
 from typing import List, Dict, Any, Tuple
 
@@ -21,7 +20,7 @@
 from qwen_omni_utils import process_mm_info
 from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor
 
-from simulstream.server.speech_processors import SAMPLE_RATE, class_load
+from simulstream.server.speech_processors import SAMPLE_RATE
 from simulstream.server.speech_processors.base_doa import DecoderOnlyAttention, get_language_name
 
 

From fcafb3002dc12c2e6da7885e0087e0044298119f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Fri, 3 Jul 2026 16:15:02 +0200
Subject: [PATCH 152/157] Address comment

---
 simulstream/server/speech_processors/base_doa.py       | 10 ++++------
 .../server/speech_processors/phi4multimodal_doa.py     |  3 ---
 simulstream/server/speech_processors/qwenomni_doa.py   |  4 ----
 3 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/simulstream/server/speech_processors/base_doa.py b/simulstream/server/speech_processors/base_doa.py
index 52cacba..a2ec6e1 100644
--- a/simulstream/server/speech_processors/base_doa.py
+++ b/simulstream/server/speech_processors/base_doa.py
@@ -54,7 +54,6 @@ class DecoderOnlyAttention(BaseStreamAtt):
         - **build_prompt**: Builds the text prompt to use with audio inputs.
         - **build_processor_inputs**: Builds model inputs from the rolling audio history.
         - **_do_generate**: Returns newly-generated tokens and self-attention scores.
-        - **tokens_to_string**: Converts decoded tokens to a plain string.
         - **_find_audio_positions**: Returns the indices of audio tokens.
 
     Args:
@@ -124,11 +123,6 @@ def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         """Return token positions corresponding to the encoded audio span."""
         ...
 
-    @abstractmethod
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        """Convert a list of decoded tokens to a plain output string."""
-        ...
-
     def _generate(self, waveform: np.ndarray) -> Tuple[List[str], torch.Tensor]:
         """
         Generate tokens from the given inputs together with the self-attention scores.
@@ -230,3 +224,7 @@ def _preprocess(self, waveform: np.float32) -> np.ndarray:
             self.audio_history = self.audio_history[-self.audio_max_len:]
 
         return self.audio_history
+
+    def tokens_to_string(self, tokens: List[str]) -> str:
+        """Convert a list of decoded tokens to a plain output string."""
+        return "".join(tokens)
diff --git a/simulstream/server/speech_processors/phi4multimodal_doa.py b/simulstream/server/speech_processors/phi4multimodal_doa.py
index 521545c..f045d95 100644
--- a/simulstream/server/speech_processors/phi4multimodal_doa.py
+++ b/simulstream/server/speech_processors/phi4multimodal_doa.py
@@ -78,9 +78,6 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
             return_tensors="pt",
         )
 
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        return "".join(tokens)
-
     def _do_generate(self, inputs: Dict[str, Any]) -> Tuple[List[str], List[torch.Tensor]]:
         input_len = inputs["input_ids"].shape[1]
         output = self.model.generate(
diff --git a/simulstream/server/speech_processors/qwenomni_doa.py b/simulstream/server/speech_processors/qwenomni_doa.py
index 15a9c12..ba8648b 100644
--- a/simulstream/server/speech_processors/qwenomni_doa.py
+++ b/simulstream/server/speech_processors/qwenomni_doa.py
@@ -120,10 +120,6 @@ def build_processor_inputs(self, waveform: np.ndarray) -> dict:
         )
         return inputs.to(self.device).to(self.model.dtype)
 
-    def tokens_to_string(self, tokens: List[str]) -> str:
-        """Convert decoded tokens to the emitted text string."""
-        return "".join(tokens)
-
     def _find_audio_positions(self, input_ids: torch.Tensor) -> torch.Tensor:
         """Return token positions corresponding to the encoded audio span."""
         audio_positions = (input_ids[0] == self.AUDIO_TOKEN_INDEX).nonzero(as_tuple=True)[0]

From ca8678ac9ff5dcd631f1b78609cb9a249e045982 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 8 Jul 2026 15:13:21 +0200
Subject: [PATCH 153/157] Address comment

---
 config/phi4multimodal_doa_fixedwords.yaml  |  16 --
 config/phi4multimodal_doa_punctuation.yaml |  15 --
 config/qwen3omni_doa_punctuation.yaml      |  21 ---
 examples/doa/README.md                     | 166 +++++++++++++++++++++
 4 files changed, 166 insertions(+), 52 deletions(-)
 delete mode 100644 config/phi4multimodal_doa_fixedwords.yaml
 delete mode 100644 config/phi4multimodal_doa_punctuation.yaml
 delete mode 100644 config/qwen3omni_doa_punctuation.yaml
 create mode 100644 examples/doa/README.md

diff --git a/config/phi4multimodal_doa_fixedwords.yaml b/config/phi4multimodal_doa_fixedwords.yaml
deleted file mode 100644
index c91b678..0000000
--- a/config/phi4multimodal_doa_fixedwords.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
-  history_words: __HISTORY__
-audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-attn_head: null  # Optional specific head; null means average over heads
-cutoff_frame_num: __FRAME__
-average_attn_over_layers: True
-detokenizer_type: "hf"
-hf_model_name: "microsoft/Phi-4-multimodal-instruct"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
-bow_prefix: " "
\ No newline at end of file
diff --git a/config/phi4multimodal_doa_punctuation.yaml b/config/phi4multimodal_doa_punctuation.yaml
deleted file mode 100644
index b54ed39..0000000
--- a/config/phi4multimodal_doa_punctuation.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 180  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-attn_head: null  # Optional specific head; null means average over heads
-cutoff_frame_num: __FRAME__
-average_attn_over_layers: True
-detokenizer_type: "hf"
-hf_model_name: "microsoft/Phi-4-multimodal-instruct"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
-bow_prefix: " "
\ No newline at end of file
diff --git a/config/qwen3omni_doa_punctuation.yaml b/config/qwen3omni_doa_punctuation.yaml
deleted file mode 100644
index 81a6b6e..0000000
--- a/config/qwen3omni_doa_punctuation.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-type: "simulstream.server.speech_processors.qwenomni_doa.Qwen3OmniDOA"
-text_history:
-  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
-audio_history_max_duration: 60  # Maximum length for the audio buffer, in seconds
-text_history_max_len: 128
-speech_chunk_size: 1  # seconds
-attn_layer: __LAYER__
-attn_head: null  # Optional specific head; null means average over heads
-average_attn_over_layers: True
-cutoff_frame_num: __FRAME__
-detokenizer_type: "hf"
-hf_model_name: "Qwen/Qwen3-Omni-30B-A3B-Instruct"
-word_level_postprocess: True  # Disable if character-level language
-max_new_tokens: 32
-bow_prefix: " "
-prompt: "You are a professional {src_lang}-to-{tgt_lang} translator. Your goal is to accurately \
-convey the meaning and nuances of the original {src_lang} speech while adhering to {tgt_lang} \
-grammar, vocabulary, and cultural sensitivities. Use precise terminology and a tone appropriate \
-for academic or instructional materials. Produce only the {tgt_lang} translation, without any \
-additional explanations or commentary. Please translate the provided {src_lang} speech into \
-{tgt_lang}:"
\ No newline at end of file
diff --git a/examples/doa/README.md b/examples/doa/README.md
new file mode 100644
index 0000000..edb2903
--- /dev/null
+++ b/examples/doa/README.md
@@ -0,0 +1,166 @@
+# Decoder-Only Attention (DOA) Policy
+
+The [**Decoder-Only Attention (DOA)** policy](https://arxiv.org/abs/2605.31432) extends the
+[encoder-decoder **StreamAtt** policy](https://aclanthology.org/2024.acl-long.202/) to SpeechLLMs
+that have no cross-attention mechanism. Instead of relying on encoder-decoder cross-attention, DOA
+builds a *proxy* cross-attention matrix by extracting the self-attention weights between the audio
+tokens and the text tokens from the decoder layers. The resulting matrix is then used by the
+[AlignAtt policy](https://www.isca-archive.org/interspeech_2023/papi23_interspeech.html) to decide
+which generated tokens can be safely emitted at each step.
+
+## Supported models
+
+| Class | HuggingFace model |
+|---|---|
+| `simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA` | `microsoft/Phi-4-multimodal-instruct` |
+| `simulstream.server.speech_processors.qwenomni_doa.Qwen3OmniDOA` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` |
+
+DOA is supported in the `simulstream_inference` backend.
+
+## Configuration
+
+A DOA config file is a YAML file passed via `--speech-processor-config`. Below is a full annotated 
+example, elements between `{}` brackets should be replaced as mentioned below:
+
+```yaml
+type: "{MODEL_CLASS}"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.{TEXT_HISTORY}"
+audio_history_max_duration: 180
+text_history_max_len: 128
+speech_chunk_size: 1              # seconds of audio fed to the model at each step
+max_new_tokens: 32                # max tokens generated per chunk
+hf_model_name: "{MODEL_NAME}"
+detokenizer_type: "hf"            
+word_level_postprocess: {WORD_POSTPROCESS}      
+bow_prefix: "{BOW_MARKER}"                   
+prompt: "{PROMPT}"
+
+# --- DOA parameters ---
+attn_layer: {ATTN_LAYER}
+attn_head: {ATTN_HEAD}
+average_attn_over_layers:  {ATTN_AVG}
+cutoff_frame_num: {CUTOFF_FRAME}               # tokens attending the last N audio frames are withheld
+```
+
+Parameters to be replaced:
+- `{MODEL_CLASS}` from the [supported models Class](#supported-models) 
+(e.g., `simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA`)
+- `{TEXT_HISTORY}` among:
+  - `FixedWordsTextHistory`: Retains the last *N* complete words. Recommended for space-separated 
+languages (English, Italian, …). In this case, the number of `history_words` should be added to 
+the config, for instance:
+  ```yaml
+  text_history:
+    type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
+    history_words: 10
+  ```
+  - `FixedCharsTextHistory`: Retains the last *N* characters. Recommended for character-level 
+languages (Chinese, Japanese) where `FixedWordsTextHistory` is ineffective because spaces are 
+sparse. n this case, the number of `history_chars` should be added to the config, for instance:
+  ```yaml
+  text_history:
+    type: "simulstream.server.speech_processors.base_streamatt.FixedWordsTextHistory"
+    history_chars: 20
+  ```
+  - `PunctuationTextHistory`: Retains the text from the last strong punctuation mark (`.`, `!`, 
+`?`, `:`, `;`, `。`). Works for both space-separated and character-level languages. 
+- `{MODEL_NAME}` from the [supported models HuggingFace model name](#supported-models) (e.g., 
+`microsoft/Phi-4-multimodal-instruct`)
+- `{WORD_POSTPROCESS}`: When `true`, the output is trimmed to complete words before emission.
+Set to `false` for character-level languages (Chinese, Japanese).
+- `{BOW_MARKER}`: The beginning-of-word (BOW) marker used by the model's tokenizer (e.g., 
+Phi-4-multimodal and Qwen3-Omni use a plain space `" "`).
+- `{PROMPT}`: User prompt. The default is `"Translate the audio to {tgt_lang}:"`. It can be 
+overridden with the `prompt` key in the yaml. The placeholders `{src_lang}` and `{tgt_lang}` are 
+filled in automatically from the language codes passed at inference time.
+- `{ATTN_LAYER}`: Decoder layer to extract self-attention from (int, 0-indexed).
+- `{ATTN_HEAD}`: Attention head to extract self-attention from (int, 0-indexed). `null` averages 
+over all heads. 
+- `{ATTN_AVG}`: If `true` (default), average the selected per-layer attention view across all 
+layers; `attn_layer` is used only when this is `false`. 
+- `{CUTOFF_FRAME}`: Cutoff frame of the AlignAtt policy. Tokens whose attention peak falls in the 
+last *N* audio frames are withheld. Higher values add more latency but reduce the risk of cutting 
+correct tokens.
+
+## Configurations of DOA's paper
+
+The configurations used to report the final results in Figure 3 are reported below:
+
+### Phi4-Multimodal
+```yaml
+type: "simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 180
+text_history_max_len: 128
+speech_chunk_size: 1 
+detokenizer_type: "hf"
+hf_model_name: "microsoft/Phi-4-multimodal-instruct"
+word_level_postprocess: True  
+max_new_tokens: 32
+bow_prefix: " "
+attn_layer: 0
+attn_head: null  
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
+```
+
+### Qwen3-Omni
+```yaml
+type: "simulstream.server.speech_processors.qwenomni_doa.Qwen3OmniDOA"
+text_history:
+  type: "simulstream.server.speech_processors.base_streamatt.PunctuationTextHistory"
+audio_history_max_duration: 60
+text_history_max_len: 128
+speech_chunk_size: 1
+detokenizer_type: "hf"
+hf_model_name: "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+word_level_postprocess: True
+max_new_tokens: 32
+bow_prefix: " "
+prompt: "You are a professional {src_lang}-to-{tgt_lang} translator. Your goal is to accurately \
+convey the meaning and nuances of the original {src_lang} speech while adhering to {tgt_lang} \
+grammar, vocabulary, and cultural sensitivities. Use precise terminology and a tone appropriate \
+for academic or instructional materials. Produce only the {tgt_lang} translation, without any \
+additional explanations or commentary. Please translate the provided {src_lang} speech into \
+{tgt_lang}:"
+attn_layer: 0
+attn_head: null
+average_attn_over_layers: True
+cutoff_frame_num: __FRAME__
+```
+To run the inference, `__FRAME__` should be replaced with `sed`:
+```bash
+simulstream_inference --speech-processor-config <(sed "s/__FRAME__/${FRAME}/g" ${CONFIG_YAML}) \
+        --wav-list-file ${AUDIOPATH_LIST} \
+        --tgt-lang $TGTLANG --src-lang en \
+        --metrics-log-file ${OUTLOG}
+```
+where `${FRAME}` is `5`, `10`, or `15` following the paper, `${CONFIG_YAML}` is the path to the 
+aforementioned configuration yaml file, `${AUDIOPATH_LIST}` is the list of test audio files path,
+and `${OUTLOG}` is the path to the jsonl output log file.
+
+## Adding a new model
+
+To support a new SpeechLLM, subclass `DecoderOnlyAttention` 
+(`simulstream.server.speech_processors.base_doa.DecoderOnlyAttention`) and implement:
+
+- `load_model(config)` — load the model and processor.
+- `build_prompt()` — return the text prompt string.
+- `build_processor_inputs(waveform)` — build processor inputs from the rolling audio history.
+- `_do_generate(inputs)` — run generation and return `(new_tokens, attentions)`.
+- `_find_audio_positions(input_ids)` — return the positions of audio tokens in the input sequence.
+
+## Citation
+If you use DOA in your work, please cite:
+
+```bibtex
+@article{papi-2026-doa,
+  title     = {{DOA}: Training-Free Decoder-Only Attention Policy for Long-Form
+               Simultaneous Translation with {SpeechLLMs}},
+  author    = {Papi, Sara and Bentivogli, Luisa},
+  journal   = {arXiv preprint arXiv:2605.31432},
+  year      = {2026},
+}
+```
\ No newline at end of file

From a933ab2961a5acc8a6e84ffd95361e2e2b866b9f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 8 Jul 2026 15:16:04 +0200
Subject: [PATCH 154/157] Remove comment from yaml

---
 examples/doa/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/doa/README.md b/examples/doa/README.md
index edb2903..ddd2f0f 100644
--- a/examples/doa/README.md
+++ b/examples/doa/README.md
@@ -40,7 +40,7 @@ prompt: "{PROMPT}"
 attn_layer: {ATTN_LAYER}
 attn_head: {ATTN_HEAD}
 average_attn_over_layers:  {ATTN_AVG}
-cutoff_frame_num: {CUTOFF_FRAME}               # tokens attending the last N audio frames are withheld
+cutoff_frame_num: {CUTOFF_FRAME}
 ```
 
 Parameters to be replaced:

From 4671fe29c6182c2dd49ffa942f57d09f0b5d470f Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 8 Jul 2026 15:25:12 +0200
Subject: [PATCH 155/157] Add default parameter for the bow_prefix

---
 simulstream/server/speech_processors/base_streamatt.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 57e4663..8e67e0b 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -273,12 +273,10 @@ class FixedWordsTextHistory:
     """
     Fixed Words textual history selection method that retains a pre-defined
     number of words in the history (*history_words*).
-
-    The current implementation supports only SentencePiece.
     """
     def __init__(self, config: SimpleNamespace, bow_prefix: str):
         self.history_words = getattr(config, "history_words", 20)
-        self.bow_prefix = bow_prefix
+        self.bow_prefix = getattr(config, "bow_prefix", "▁pytest uts/")
         self.config = config
 
     def select_text_history(self, text_history: List[str]):

From 52bafd205a49c448b9cec4a57b7de83764c0cddc Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 8 Jul 2026 15:40:29 +0200
Subject: [PATCH 156/157] fix bow_prefix leftovers

---
 simulstream/server/speech_processors/base_streamatt.py | 6 +++---
 uts/speech_processors/test_streamatt.py                | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/simulstream/server/speech_processors/base_streamatt.py b/simulstream/server/speech_processors/base_streamatt.py
index 1a1c698..b5c5128 100644
--- a/simulstream/server/speech_processors/base_streamatt.py
+++ b/simulstream/server/speech_processors/base_streamatt.py
@@ -274,9 +274,9 @@ class FixedWordsTextHistory:
     Fixed Words textual history selection method that retains a pre-defined
     number of words in the history (*history_words*).
     """
-    def __init__(self, config: SimpleNamespace, bow_prefix: str):
+    def __init__(self, config: SimpleNamespace):
         self.history_words = getattr(config, "history_words", 20)
-        self.bow_prefix = getattr(config, "bow_prefix", "▁pytest uts/")
+        self.bow_prefix = getattr(config, "bow_prefix", BOW_PREFIX)
         self.config = config
 
     def select_text_history(self, text_history: List[str]):
@@ -327,7 +327,7 @@ class PunctuationTextHistory:
 
     STRONG_PUNCTUATION = [".", "!", "?", ":", ";", "。"]
 
-    def __init__(self, config: SimpleNamespace, bow_prefix: str):
+    def __init__(self, config: SimpleNamespace):
         self.config = config
 
     def select_text_history(self, text_history):
diff --git a/uts/speech_processors/test_streamatt.py b/uts/speech_processors/test_streamatt.py
index 75ca1e2..a20878f 100644
--- a/uts/speech_processors/test_streamatt.py
+++ b/uts/speech_processors/test_streamatt.py
@@ -59,7 +59,7 @@ def test_shorter_than_limit(self):
 class TestPunctuationTextHistory(unittest.TestCase):
     def setUp(self):
         self.config = SimpleNamespace()
-        self.punctuation_text_history = PunctuationTextHistory(self.config, "")
+        self.punctuation_text_history = PunctuationTextHistory(self.config)
 
     def test_punctuation_last(self):
         """ Test PunctuationTextHistory method when the history ends with strong punctuation. """

From 80fe71256322c84b9bc4e233c33a44becfd295c0 Mon Sep 17 00:00:00 2001
From: spapi <spapi@fbk.eu>
Date: Wed, 8 Jul 2026 19:07:25 +0200
Subject: [PATCH 157/157] Address comment

---
 examples/doa/README.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/examples/doa/README.md b/examples/doa/README.md
index ddd2f0f..e836c90 100644
--- a/examples/doa/README.md
+++ b/examples/doa/README.md
@@ -15,8 +15,6 @@ which generated tokens can be safely emitted at each step.
 | `simulstream.server.speech_processors.phi4multimodal_doa.Phi4MultimodalDOA` | `microsoft/Phi-4-multimodal-instruct` |
 | `simulstream.server.speech_processors.qwenomni_doa.Qwen3OmniDOA` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` |
 
-DOA is supported in the `simulstream_inference` backend.
-
 ## Configuration
 
 A DOA config file is a YAML file passed via `--speech-processor-config`. Below is a full annotated