#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2026 The OpenBMB Team. All rights reserved. # Modifications Copyright 2026 The Realtime-Venus Team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from typing import Union from transformers import PretrainedConfig from transformers import Qwen3Config from transformers import WhisperConfig from transformers.utils import logging from .modeling_navit_siglip import SiglipVisionConfig logger = logging.get_logger(__name__) class MiniCPMVSliceConfig(PretrainedConfig): model_type = "minicpmv" def __init__( self, patch_size=14, max_slice_nums=9, scale_resolution=448, **kwargs, ): super().__init__(**kwargs) self.patch_size = patch_size self.max_slice_nums = max_slice_nums self.scale_resolution = scale_resolution @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": cls._set_token_in_kwargs(kwargs) config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) if config_dict.get("model_type") == "minicpmv": config_dict = config_dict["slice_config"] if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: logger.warning( f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." ) return cls.from_dict(config_dict, **kwargs) class MiniCPMTTSConfig(PretrainedConfig): model_type = "minicpmtts" def __init__( self, llm_dim: int = 2560, llm_intermediate_size: int = 768, llm_down_scale: bool = False, llm_dim_model_base: int = 256, projector_type: str = "mlp", hidden_act: str = "silu", aug_loss_weight: bool = False, aug_layer_loss_weight: bool = False, filter_tts_loss: bool = False, tts_filter_loss_fix: bool = False, long_weight: float = 0.1, short_weight: float = 0.1, hidden_size: int = 768, intermediate_size: int = 3072, num_attention_heads: int = 12, num_hidden_layers: int = 20, num_key_value_heads: int = 12, max_position_embeddings: int = 4096, num_audio_tokens: int = 4097, num_text_tokens: int = 21178, num_mel_bins: int = 100, num_vq: int = 1, use_llm_hidden_state: bool = False, audio_bos_token_id: int = 21132, text_eos_token_id: int = 21133, use_text: bool = True, streaming: bool = False, streaming_text_chunk_min: int = 3, streaming_text_chunk_max: int = 7, streaming_text_reserved_len: int = 300, streaming_audio_chunk_size: int = 50, attn_implementation: str = "sdpa", condition_type: str = "llm_hidden", backbone_model: str = "llama", audio_tokenizer_type: str = "wavtokenizer", audio_tokenizer_sample_rate: int = 24000, streaming_sliding_window: bool = False, streaming_sliding_window_max_text_len: int = 500, streaming_sliding_window_average_speed: int = 5, streaming_sliding_window_fast_speed: int = 7, streaming_sliding_window_slow_speed: int = 3, streaming_sliding_window_audio_frame_rate: int = 50, streaming_sliding_window_audio_init_text_length: int = 10, streaming_sliding_window_audio_window_size: int = 300, normalize_projected_hidden: bool = False, interleaved: bool = False, attention_type: str = "sliding_recompute", recomputed_chunks: int = 1, window_size: int = 2, **kwargs, ): super().__init__(**kwargs) self.llm_dim = llm_dim self.llm_hidden_size = llm_dim self.llm_intermediate_size = llm_intermediate_size self.llm_down_scale = llm_down_scale self.llm_dim_model_base = llm_dim_model_base self.projector_type = projector_type self.aug_loss_weight = aug_loss_weight self.aug_layer_loss_weight = aug_layer_loss_weight self.tts_filter_loss_fix = tts_filter_loss_fix self.filter_tts_loss = filter_tts_loss self.long_weight = long_weight self.short_weight = short_weight self.hidden_act = hidden_act self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.num_attention_heads = num_attention_heads self.num_hidden_layers = num_hidden_layers self.num_key_value_heads = num_key_value_heads self.max_position_embeddings = max_position_embeddings self.num_audio_tokens = num_audio_tokens self.num_text_tokens = num_text_tokens self.num_mel_bins = num_mel_bins self.num_vq = num_vq self.use_llm_hidden_state = use_llm_hidden_state self.audio_bos_token_id = audio_bos_token_id self.text_eos_token_id = text_eos_token_id self.use_text = use_text self.streaming = streaming self.streaming_text_chunk_min = streaming_text_chunk_min self.streaming_text_chunk_max = streaming_text_chunk_max self.streaming_text_reserved_len = streaming_text_reserved_len self.streaming_audio_chunk_size = streaming_audio_chunk_size self.attn_implementation = attn_implementation self.condition_type = condition_type self.backbone_model = backbone_model self.audio_tokenizer_type = audio_tokenizer_type self.audio_tokenizer_sample_rate = audio_tokenizer_sample_rate self.streaming_sliding_window = streaming_sliding_window self.streaming_sliding_window_max_text_len = streaming_sliding_window_max_text_len self.streaming_sliding_window_average_speed = streaming_sliding_window_average_speed self.streaming_sliding_window_fast_speed = streaming_sliding_window_fast_speed self.streaming_sliding_window_slow_speed = streaming_sliding_window_slow_speed self.streaming_sliding_window_audio_frame_rate = streaming_sliding_window_audio_frame_rate self.streaming_sliding_window_audio_init_text_length = streaming_sliding_window_audio_init_text_length self.streaming_sliding_window_audio_window_size = streaming_sliding_window_audio_window_size self.normalize_projected_hidden = normalize_projected_hidden self.interleaved = interleaved self.attention_type = attention_type self.recomputed_chunks = recomputed_chunks self.window_size = window_size class MiniCPMOConfig(Qwen3Config): model_type = "minicpmo" keys_to_ignore_at_inference = ["past_key_values"] default_vision_config = { "hidden_size": 1152, "image_size": 980, "intermediate_size": 4304, "model_type": "siglip", "num_attention_heads": 16, "num_hidden_layers": 27, "patch_size": 14, } def __init__( self, use_cache=True, query_num=64, image_size=448, drop_vision_last_layer=True, batch_vision_input=True, slice_config=None, vision_config=None, audio_config=None, tts_config=None, use_image_id=True, vision_batch_size=16, audio_pool_step=5, audio_chunk_length=1.0, stream_input=False, listen_speak_type="asr", init_vision=True, init_audio=True, init_tts=True, **kwargs, ): self.use_cache = use_cache self.query_num = query_num self.image_size = image_size self.drop_vision_last_layer = drop_vision_last_layer self.batch_vision_input = batch_vision_input self.use_image_id = use_image_id self.vision_batch_size = vision_batch_size self.audio_pool_step = audio_pool_step self.audio_chunk_length = audio_chunk_length self.stream_input = stream_input self.listen_speak_type = listen_speak_type self.init_vision = init_vision self.init_audio = init_audio self.init_tts = init_tts if slice_config is None: self.slice_config = MiniCPMVSliceConfig(max_slice_nums=1) else: self.slice_config = MiniCPMVSliceConfig(**slice_config) self.slice_mode = True # same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit add tgt_sizes if vision_config is None: self.vision_config = SiglipVisionConfig(**self.default_vision_config) logger.info("vision_config is None, using default vision config") elif isinstance(vision_config, dict): self.vision_config = SiglipVisionConfig(**vision_config) elif isinstance(vision_config, SiglipVisionConfig): self.vision_config = vision_config if audio_config is None: self.audio_config = WhisperConfig() elif isinstance(audio_config, dict): self.audio_config = WhisperConfig(**audio_config) elif isinstance(audio_config, WhisperConfig): self.audio_config = audio_config if tts_config is None: self.tts_config = MiniCPMTTSConfig() elif isinstance(tts_config, dict): self.tts_config = MiniCPMTTSConfig(**tts_config) elif isinstance(tts_config, MiniCPMTTSConfig): self.tts_config = tts_config self.patch_size = self.vision_config.patch_size super().__init__(**kwargs) @classmethod def get_config_dict(cls, pretrained_model_name_or_path, **kwargs): options = dict(kwargs) cls._set_token_in_kwargs(options) config, remaining = super().get_config_dict(pretrained_model_name_or_path, **kwargs) return _expand_venus_config(config, pretrained_model_name_or_path, options, 'audio'), remaining """This helper is embedded into each existing model configuration module.""" def _expand_venus_config(config, model_path, options, expected_model): import copy import hashlib import re from pathlib import Path ref = config.get('venus_config_ref') if ref is None: return config expected_types = {'audio': 'minicpmo', 'omni': 'realtime_venus_omni'} if not isinstance(ref, dict) or set(ref) != {'repo_id','repo_type','filename','revision','sha256','model'}: raise ValueError('Invalid venus_config_ref fields') if (ref['repo_id'] != 'inclusionAI/Realtime-Venus' or ref['repo_type'] != 'model' or ref['filename'] != 'config.yaml' or ref['model'] != expected_model): raise ValueError('venus_config_ref repository/file/model identity mismatch') if not isinstance(ref['revision'],str) or not re.fullmatch(r'[0-9a-f]{40}', ref['revision']): raise ValueError('venus_config_ref requires a fixed full commit SHA') if not isinstance(ref['sha256'],str) or not re.fullmatch(r'[0-9a-f]{64}', ref['sha256']): raise ValueError('venus_config_ref requires a SHA-256 digest') if config.get('model_type') != expected_types[expected_model]: raise ValueError('Child configuration model_type mismatch') raw = None path = Path(model_path) subfolder = options.get('subfolder', '') candidates = [] if path.is_dir(): model_dir = path / subfolder if subfolder else path candidates.append(model_dir.parent / 'config.yaml') elif path.is_file(): candidates.append(path.parent.parent / 'config.yaml') for candidate in candidates: if candidate.is_file(): value = candidate.read_bytes() if hashlib.sha256(value).hexdigest() == ref['sha256']: raw = value break if raw is None: from huggingface_hub import hf_hub_download, try_to_load_from_cache from transformers.utils.hub import TRANSFORMERS_CACHE cache_dir = options.get('cache_dir') or TRANSFORMERS_CACHE cached = try_to_load_from_cache(ref['repo_id'],ref['filename'], revision=ref['revision'],repo_type='model',cache_dir=cache_dir) if isinstance(cached,str): raw = Path(cached).read_bytes() else: download_options = {k: options[k] for k in ('token','local_files_only','force_download','user_agent','proxies') if k in options} download_options['cache_dir'] = cache_dir try: resolved = hf_hub_download(repo_id=ref['repo_id'],repo_type='model', filename=ref['filename'],revision=ref['revision'],**download_options) except OSError as error: raise OSError('Required pinned Realtime-Venus config.yaml is unavailable locally/in cache; ' 'download that revision while online or place its verified YAML beside the model directory.') from error raw = Path(resolved).read_bytes() if hashlib.sha256(raw).hexdigest() != ref['sha256']: raise ValueError('Pinned Realtime-Venus config.yaml SHA-256 mismatch') import yaml document = yaml.safe_load(raw) if not isinstance(document,dict) or document.get('schema_version') != 3 or document.get('name') != 'Realtime-Venus': raise ValueError('Invalid Realtime-Venus root YAML schema/name') models = document.get('models') entry = models.get(expected_model) if isinstance(models,dict) else None expected_path = 'Realtime-Venus-Audio' if expected_model == 'audio' else 'Realtime-Venus-Omni' if not isinstance(entry,dict) or entry.get('path') != expected_path: raise ValueError('Root YAML model path mismatch') full = entry.get('hf_config') if not isinstance(full,dict) or 'venus_config_ref' in full or full.get('model_type') != expected_types[expected_model]: raise ValueError('Root YAML hf_config model identity mismatch') if full.get('architectures') != config.get('architectures'): raise ValueError('Root YAML architecture mismatch') for field in ('audio_config','vision_config','tts_config'): if not isinstance(full.get(field),dict): raise ValueError('Incomplete root hf_config: ' + field) def auto_map_identity(mapping): if not isinstance(mapping,dict): raise ValueError('Invalid auto_map') return {k:v.split('--')[-1] for k,v in mapping.items()} if auto_map_identity(full.get('auto_map')) != auto_map_identity(config.get('auto_map')): raise ValueError('Root YAML AutoClass identity mismatch') expanded = copy.deepcopy(full) # Keep the current model snapshot's code/weights revision, not the YAML's. if '_commit_hash' in config: expanded['_commit_hash'] = config['_commit_hash'] expanded['auto_map'] = copy.deepcopy(config['auto_map']) expanded['_name_or_path'] = str(model_path) return expanded