File size: 15,534 Bytes
04bceb2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
a1d95d3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2026 The OpenBMB Team. All rights reserved.
# Modifications Copyright 2026 The Realtime-Venus Team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from typing import Union

from transformers import PretrainedConfig
from transformers import Qwen3Config
from transformers import WhisperConfig
from transformers.utils import logging

from .modeling_navit_siglip import SiglipVisionConfig

logger = logging.get_logger(__name__)


class MiniCPMVSliceConfig(PretrainedConfig):
    model_type = "minicpmv"

    def __init__(
        self,
        patch_size=14,
        max_slice_nums=9,
        scale_resolution=448,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.patch_size = patch_size
        self.max_slice_nums = max_slice_nums
        self.scale_resolution = scale_resolution

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        if config_dict.get("model_type") == "minicpmv":
            config_dict = config_dict["slice_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class MiniCPMTTSConfig(PretrainedConfig):
    model_type = "minicpmtts"

    def __init__(
        self,
        llm_dim: int = 2560,
        llm_intermediate_size: int = 768,
        llm_down_scale: bool = False,
        llm_dim_model_base: int = 256,
        projector_type: str = "mlp",
        hidden_act: str = "silu",
        aug_loss_weight: bool = False,
        aug_layer_loss_weight: bool = False,
        filter_tts_loss: bool = False,
        tts_filter_loss_fix: bool = False,
        long_weight: float = 0.1,
        short_weight: float = 0.1,
        hidden_size: int = 768,
        intermediate_size: int = 3072,
        num_attention_heads: int = 12,
        num_hidden_layers: int = 20,
        num_key_value_heads: int = 12,
        max_position_embeddings: int = 4096,
        num_audio_tokens: int = 4097,
        num_text_tokens: int = 21178,
        num_mel_bins: int = 100,
        num_vq: int = 1,
        use_llm_hidden_state: bool = False,
        audio_bos_token_id: int = 21132,
        text_eos_token_id: int = 21133,
        use_text: bool = True,
        streaming: bool = False,
        streaming_text_chunk_min: int = 3,
        streaming_text_chunk_max: int = 7,
        streaming_text_reserved_len: int = 300,
        streaming_audio_chunk_size: int = 50,
        attn_implementation: str = "sdpa",
        condition_type: str = "llm_hidden",
        backbone_model: str = "llama",
        audio_tokenizer_type: str = "wavtokenizer",
        audio_tokenizer_sample_rate: int = 24000,
        streaming_sliding_window: bool = False,
        streaming_sliding_window_max_text_len: int = 500,
        streaming_sliding_window_average_speed: int = 5,
        streaming_sliding_window_fast_speed: int = 7,
        streaming_sliding_window_slow_speed: int = 3,
        streaming_sliding_window_audio_frame_rate: int = 50,
        streaming_sliding_window_audio_init_text_length: int = 10,
        streaming_sliding_window_audio_window_size: int = 300,
        normalize_projected_hidden: bool = False,
        interleaved: bool = False,
        attention_type: str = "sliding_recompute",
        recomputed_chunks: int = 1,
        window_size: int = 2,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.llm_dim = llm_dim
        self.llm_hidden_size = llm_dim
        self.llm_intermediate_size = llm_intermediate_size
        self.llm_down_scale = llm_down_scale
        self.llm_dim_model_base = llm_dim_model_base
        self.projector_type = projector_type
        self.aug_loss_weight = aug_loss_weight
        self.aug_layer_loss_weight = aug_layer_loss_weight
        self.tts_filter_loss_fix = tts_filter_loss_fix
        self.filter_tts_loss = filter_tts_loss
        self.long_weight = long_weight
        self.short_weight = short_weight
        self.hidden_act = hidden_act

        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_attention_heads = num_attention_heads
        self.num_hidden_layers = num_hidden_layers
        self.num_key_value_heads = num_key_value_heads
        self.max_position_embeddings = max_position_embeddings
        self.num_audio_tokens = num_audio_tokens
        self.num_text_tokens = num_text_tokens
        self.num_mel_bins = num_mel_bins
        self.num_vq = num_vq
        self.use_llm_hidden_state = use_llm_hidden_state
        self.audio_bos_token_id = audio_bos_token_id
        self.text_eos_token_id = text_eos_token_id
        self.use_text = use_text
        self.streaming = streaming
        self.streaming_text_chunk_min = streaming_text_chunk_min
        self.streaming_text_chunk_max = streaming_text_chunk_max
        self.streaming_text_reserved_len = streaming_text_reserved_len
        self.streaming_audio_chunk_size = streaming_audio_chunk_size
        self.attn_implementation = attn_implementation
        self.condition_type = condition_type
        self.backbone_model = backbone_model
        self.audio_tokenizer_type = audio_tokenizer_type
        self.audio_tokenizer_sample_rate = audio_tokenizer_sample_rate

        self.streaming_sliding_window = streaming_sliding_window
        self.streaming_sliding_window_max_text_len = streaming_sliding_window_max_text_len
        self.streaming_sliding_window_average_speed = streaming_sliding_window_average_speed
        self.streaming_sliding_window_fast_speed = streaming_sliding_window_fast_speed
        self.streaming_sliding_window_slow_speed = streaming_sliding_window_slow_speed
        self.streaming_sliding_window_audio_frame_rate = streaming_sliding_window_audio_frame_rate
        self.streaming_sliding_window_audio_init_text_length = streaming_sliding_window_audio_init_text_length
        self.streaming_sliding_window_audio_window_size = streaming_sliding_window_audio_window_size

        self.normalize_projected_hidden = normalize_projected_hidden

        self.interleaved = interleaved
        self.attention_type = attention_type
        self.recomputed_chunks = recomputed_chunks
        self.window_size = window_size


class MiniCPMOConfig(Qwen3Config):
    model_type = "minicpmo"
    keys_to_ignore_at_inference = ["past_key_values"]

    default_vision_config = {
        "hidden_size": 1152,
        "image_size": 980,
        "intermediate_size": 4304,
        "model_type": "siglip",
        "num_attention_heads": 16,
        "num_hidden_layers": 27,
        "patch_size": 14,
    }

    def __init__(
        self,
        use_cache=True,
        query_num=64,
        image_size=448,
        drop_vision_last_layer=True,
        batch_vision_input=True,
        slice_config=None,
        vision_config=None,
        audio_config=None,
        tts_config=None,
        use_image_id=True,
        vision_batch_size=16,
        audio_pool_step=5,
        audio_chunk_length=1.0,
        stream_input=False,
        listen_speak_type="asr",
        init_vision=True,
        init_audio=True,
        init_tts=True,
        **kwargs,
    ):
        self.use_cache = use_cache
        self.query_num = query_num
        self.image_size = image_size
        self.drop_vision_last_layer = drop_vision_last_layer
        self.batch_vision_input = batch_vision_input
        self.use_image_id = use_image_id
        self.vision_batch_size = vision_batch_size
        self.audio_pool_step = audio_pool_step
        self.audio_chunk_length = audio_chunk_length
        self.stream_input = stream_input
        self.listen_speak_type = listen_speak_type

        self.init_vision = init_vision
        self.init_audio = init_audio
        self.init_tts = init_tts

        if slice_config is None:
            self.slice_config = MiniCPMVSliceConfig(max_slice_nums=1)
        else:
            self.slice_config = MiniCPMVSliceConfig(**slice_config)
        self.slice_mode = True

        # same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit add tgt_sizes
        if vision_config is None:
            self.vision_config = SiglipVisionConfig(**self.default_vision_config)
            logger.info("vision_config is None, using default vision config")
        elif isinstance(vision_config, dict):
            self.vision_config = SiglipVisionConfig(**vision_config)
        elif isinstance(vision_config, SiglipVisionConfig):
            self.vision_config = vision_config

        if audio_config is None:
            self.audio_config = WhisperConfig()
        elif isinstance(audio_config, dict):
            self.audio_config = WhisperConfig(**audio_config)
        elif isinstance(audio_config, WhisperConfig):
            self.audio_config = audio_config

        if tts_config is None:
            self.tts_config = MiniCPMTTSConfig()
        elif isinstance(tts_config, dict):
            self.tts_config = MiniCPMTTSConfig(**tts_config)
        elif isinstance(tts_config, MiniCPMTTSConfig):
            self.tts_config = tts_config

        self.patch_size = self.vision_config.patch_size

        super().__init__(**kwargs)

    @classmethod
    def get_config_dict(cls, pretrained_model_name_or_path, **kwargs):
        options = dict(kwargs)
        cls._set_token_in_kwargs(options)
        config, remaining = super().get_config_dict(pretrained_model_name_or_path, **kwargs)
        return _expand_venus_config(config, pretrained_model_name_or_path, options, 'audio'), remaining


"""This helper is embedded into each existing model configuration module."""


def _expand_venus_config(config, model_path, options, expected_model):
    import copy
    import hashlib
    import re
    from pathlib import Path

    ref = config.get('venus_config_ref')
    if ref is None:
        return config
    expected_types = {'audio': 'minicpmo', 'omni': 'realtime_venus_omni'}
    if not isinstance(ref, dict) or set(ref) != {'repo_id','repo_type','filename','revision','sha256','model'}:
        raise ValueError('Invalid venus_config_ref fields')
    if (ref['repo_id'] != 'inclusionAI/Realtime-Venus' or ref['repo_type'] != 'model'
            or ref['filename'] != 'config.yaml' or ref['model'] != expected_model):
        raise ValueError('venus_config_ref repository/file/model identity mismatch')
    if not isinstance(ref['revision'],str) or not re.fullmatch(r'[0-9a-f]{40}', ref['revision']):
        raise ValueError('venus_config_ref requires a fixed full commit SHA')
    if not isinstance(ref['sha256'],str) or not re.fullmatch(r'[0-9a-f]{64}', ref['sha256']):
        raise ValueError('venus_config_ref requires a SHA-256 digest')
    if config.get('model_type') != expected_types[expected_model]:
        raise ValueError('Child configuration model_type mismatch')

    raw = None
    path = Path(model_path)
    subfolder = options.get('subfolder', '')
    candidates = []
    if path.is_dir():
        model_dir = path / subfolder if subfolder else path
        candidates.append(model_dir.parent / 'config.yaml')
    elif path.is_file():
        candidates.append(path.parent.parent / 'config.yaml')
    for candidate in candidates:
        if candidate.is_file():
            value = candidate.read_bytes()
            if hashlib.sha256(value).hexdigest() == ref['sha256']:
                raw = value
                break

    if raw is None:
        from huggingface_hub import hf_hub_download, try_to_load_from_cache
        from transformers.utils.hub import TRANSFORMERS_CACHE
        cache_dir = options.get('cache_dir') or TRANSFORMERS_CACHE
        cached = try_to_load_from_cache(ref['repo_id'],ref['filename'],
            revision=ref['revision'],repo_type='model',cache_dir=cache_dir)
        if isinstance(cached,str):
            raw = Path(cached).read_bytes()
        else:
            download_options = {k: options[k] for k in
                ('token','local_files_only','force_download','user_agent','proxies') if k in options}
            download_options['cache_dir'] = cache_dir
            try:
                resolved = hf_hub_download(repo_id=ref['repo_id'],repo_type='model',
                    filename=ref['filename'],revision=ref['revision'],**download_options)
            except OSError as error:
                raise OSError('Required pinned Realtime-Venus config.yaml is unavailable locally/in cache; '
                    'download that revision while online or place its verified YAML beside the model directory.') from error
            raw = Path(resolved).read_bytes()
    if hashlib.sha256(raw).hexdigest() != ref['sha256']:
        raise ValueError('Pinned Realtime-Venus config.yaml SHA-256 mismatch')

    import yaml
    document = yaml.safe_load(raw)
    if not isinstance(document,dict) or document.get('schema_version') != 3 or document.get('name') != 'Realtime-Venus':
        raise ValueError('Invalid Realtime-Venus root YAML schema/name')
    models = document.get('models')
    entry = models.get(expected_model) if isinstance(models,dict) else None
    expected_path = 'Realtime-Venus-Audio' if expected_model == 'audio' else 'Realtime-Venus-Omni'
    if not isinstance(entry,dict) or entry.get('path') != expected_path:
        raise ValueError('Root YAML model path mismatch')
    full = entry.get('hf_config')
    if not isinstance(full,dict) or 'venus_config_ref' in full or full.get('model_type') != expected_types[expected_model]:
        raise ValueError('Root YAML hf_config model identity mismatch')
    if full.get('architectures') != config.get('architectures'):
        raise ValueError('Root YAML architecture mismatch')
    for field in ('audio_config','vision_config','tts_config'):
        if not isinstance(full.get(field),dict):
            raise ValueError('Incomplete root hf_config: ' + field)
    def auto_map_identity(mapping):
        if not isinstance(mapping,dict):
            raise ValueError('Invalid auto_map')
        return {k:v.split('--')[-1] for k,v in mapping.items()}
    if auto_map_identity(full.get('auto_map')) != auto_map_identity(config.get('auto_map')):
        raise ValueError('Root YAML AutoClass identity mismatch')
    expanded = copy.deepcopy(full)
    # Keep the current model snapshot's code/weights revision, not the YAML's.
    if '_commit_hash' in config:
        expanded['_commit_hash'] = config['_commit_hash']
    expanded['auto_map'] = copy.deepcopy(config['auto_map'])
    expanded['_name_or_path'] = str(model_path)
    return expanded