Realtime-Venus / Realtime-Venus-Audio /configuration_minicpmo.py
Codex
Publish reviewed Realtime-Venus configurations, HEARTBEAT index and documentation
a1d95d3
Raw History Blame Contribute Delete
15.5 kB
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2026 The OpenBMB Team. All rights reserved.
# Modifications Copyright 2026 The Realtime-Venus Team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import Union
from transformers import PretrainedConfig
from transformers import Qwen3Config
from transformers import WhisperConfig
from transformers.utils import logging
from .modeling_navit_siglip import SiglipVisionConfig
logger = logging.get_logger(__name__)
class MiniCPMVSliceConfig(PretrainedConfig):
model_type = "minicpmv"
def __init__(
self,
patch_size=14,
max_slice_nums=9,
scale_resolution=448,
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
self.max_slice_nums = max_slice_nums
self.scale_resolution = scale_resolution
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
cls._set_token_in_kwargs(kwargs)
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
if config_dict.get("model_type") == "minicpmv":
config_dict = config_dict["slice_config"]
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
logger.warning(
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
)
return cls.from_dict(config_dict, **kwargs)
class MiniCPMTTSConfig(PretrainedConfig):
model_type = "minicpmtts"
def __init__(
self,
llm_dim: int = 2560,
llm_intermediate_size: int = 768,
llm_down_scale: bool = False,
llm_dim_model_base: int = 256,
projector_type: str = "mlp",
hidden_act: str = "silu",
aug_loss_weight: bool = False,
aug_layer_loss_weight: bool = False,
filter_tts_loss: bool = False,
tts_filter_loss_fix: bool = False,
long_weight: float = 0.1,
short_weight: float = 0.1,
hidden_size: int = 768,
intermediate_size: int = 3072,
num_attention_heads: int = 12,
num_hidden_layers: int = 20,
num_key_value_heads: int = 12,
max_position_embeddings: int = 4096,
num_audio_tokens: int = 4097,
num_text_tokens: int = 21178,
num_mel_bins: int = 100,
num_vq: int = 1,
use_llm_hidden_state: bool = False,
audio_bos_token_id: int = 21132,
text_eos_token_id: int = 21133,
use_text: bool = True,
streaming: bool = False,
streaming_text_chunk_min: int = 3,
streaming_text_chunk_max: int = 7,
streaming_text_reserved_len: int = 300,
streaming_audio_chunk_size: int = 50,
attn_implementation: str = "sdpa",
condition_type: str = "llm_hidden",
backbone_model: str = "llama",
audio_tokenizer_type: str = "wavtokenizer",
audio_tokenizer_sample_rate: int = 24000,
streaming_sliding_window: bool = False,
streaming_sliding_window_max_text_len: int = 500,
streaming_sliding_window_average_speed: int = 5,
streaming_sliding_window_fast_speed: int = 7,
streaming_sliding_window_slow_speed: int = 3,
streaming_sliding_window_audio_frame_rate: int = 50,
streaming_sliding_window_audio_init_text_length: int = 10,
streaming_sliding_window_audio_window_size: int = 300,
normalize_projected_hidden: bool = False,
interleaved: bool = False,
attention_type: str = "sliding_recompute",
recomputed_chunks: int = 1,
window_size: int = 2,
**kwargs,
):
super().__init__(**kwargs)
self.llm_dim = llm_dim
self.llm_hidden_size = llm_dim
self.llm_intermediate_size = llm_intermediate_size
self.llm_down_scale = llm_down_scale
self.llm_dim_model_base = llm_dim_model_base
self.projector_type = projector_type
self.aug_loss_weight = aug_loss_weight
self.aug_layer_loss_weight = aug_layer_loss_weight
self.tts_filter_loss_fix = tts_filter_loss_fix
self.filter_tts_loss = filter_tts_loss
self.long_weight = long_weight
self.short_weight = short_weight
self.hidden_act = hidden_act
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_attention_heads = num_attention_heads
self.num_hidden_layers = num_hidden_layers
self.num_key_value_heads = num_key_value_heads
self.max_position_embeddings = max_position_embeddings
self.num_audio_tokens = num_audio_tokens
self.num_text_tokens = num_text_tokens
self.num_mel_bins = num_mel_bins
self.num_vq = num_vq
self.use_llm_hidden_state = use_llm_hidden_state
self.audio_bos_token_id = audio_bos_token_id
self.text_eos_token_id = text_eos_token_id
self.use_text = use_text
self.streaming = streaming
self.streaming_text_chunk_min = streaming_text_chunk_min
self.streaming_text_chunk_max = streaming_text_chunk_max
self.streaming_text_reserved_len = streaming_text_reserved_len
self.streaming_audio_chunk_size = streaming_audio_chunk_size
self.attn_implementation = attn_implementation
self.condition_type = condition_type
self.backbone_model = backbone_model
self.audio_tokenizer_type = audio_tokenizer_type
self.audio_tokenizer_sample_rate = audio_tokenizer_sample_rate
self.streaming_sliding_window = streaming_sliding_window
self.streaming_sliding_window_max_text_len = streaming_sliding_window_max_text_len
self.streaming_sliding_window_average_speed = streaming_sliding_window_average_speed
self.streaming_sliding_window_fast_speed = streaming_sliding_window_fast_speed
self.streaming_sliding_window_slow_speed = streaming_sliding_window_slow_speed
self.streaming_sliding_window_audio_frame_rate = streaming_sliding_window_audio_frame_rate
self.streaming_sliding_window_audio_init_text_length = streaming_sliding_window_audio_init_text_length
self.streaming_sliding_window_audio_window_size = streaming_sliding_window_audio_window_size
self.normalize_projected_hidden = normalize_projected_hidden
self.interleaved = interleaved
self.attention_type = attention_type
self.recomputed_chunks = recomputed_chunks
self.window_size = window_size
class MiniCPMOConfig(Qwen3Config):
model_type = "minicpmo"
keys_to_ignore_at_inference = ["past_key_values"]
default_vision_config = {
"hidden_size": 1152,
"image_size": 980,
"intermediate_size": 4304,
"model_type": "siglip",
"num_attention_heads": 16,
"num_hidden_layers": 27,
"patch_size": 14,
}
def __init__(
self,
use_cache=True,
query_num=64,
image_size=448,
drop_vision_last_layer=True,
batch_vision_input=True,
slice_config=None,
vision_config=None,
audio_config=None,
tts_config=None,
use_image_id=True,
vision_batch_size=16,
audio_pool_step=5,
audio_chunk_length=1.0,
stream_input=False,
listen_speak_type="asr",
init_vision=True,
init_audio=True,
init_tts=True,
**kwargs,
):
self.use_cache = use_cache
self.query_num = query_num
self.image_size = image_size
self.drop_vision_last_layer = drop_vision_last_layer
self.batch_vision_input = batch_vision_input
self.use_image_id = use_image_id
self.vision_batch_size = vision_batch_size
self.audio_pool_step = audio_pool_step
self.audio_chunk_length = audio_chunk_length
self.stream_input = stream_input
self.listen_speak_type = listen_speak_type
self.init_vision = init_vision
self.init_audio = init_audio
self.init_tts = init_tts
if slice_config is None:
self.slice_config = MiniCPMVSliceConfig(max_slice_nums=1)
else:
self.slice_config = MiniCPMVSliceConfig(**slice_config)
self.slice_mode = True
# same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit add tgt_sizes
if vision_config is None:
self.vision_config = SiglipVisionConfig(**self.default_vision_config)
logger.info("vision_config is None, using default vision config")
elif isinstance(vision_config, dict):
self.vision_config = SiglipVisionConfig(**vision_config)
elif isinstance(vision_config, SiglipVisionConfig):
self.vision_config = vision_config
if audio_config is None:
self.audio_config = WhisperConfig()
elif isinstance(audio_config, dict):
self.audio_config = WhisperConfig(**audio_config)
elif isinstance(audio_config, WhisperConfig):
self.audio_config = audio_config
if tts_config is None:
self.tts_config = MiniCPMTTSConfig()
elif isinstance(tts_config, dict):
self.tts_config = MiniCPMTTSConfig(**tts_config)
elif isinstance(tts_config, MiniCPMTTSConfig):
self.tts_config = tts_config
self.patch_size = self.vision_config.patch_size
super().__init__(**kwargs)
@classmethod
def get_config_dict(cls, pretrained_model_name_or_path, **kwargs):
options = dict(kwargs)
cls._set_token_in_kwargs(options)
config, remaining = super().get_config_dict(pretrained_model_name_or_path, **kwargs)
return _expand_venus_config(config, pretrained_model_name_or_path, options, 'audio'), remaining
"""This helper is embedded into each existing model configuration module."""
def _expand_venus_config(config, model_path, options, expected_model):
import copy
import hashlib
import re
from pathlib import Path
ref = config.get('venus_config_ref')
if ref is None:
return config
expected_types = {'audio': 'minicpmo', 'omni': 'realtime_venus_omni'}
if not isinstance(ref, dict) or set(ref) != {'repo_id','repo_type','filename','revision','sha256','model'}:
raise ValueError('Invalid venus_config_ref fields')
if (ref['repo_id'] != 'inclusionAI/Realtime-Venus' or ref['repo_type'] != 'model'
or ref['filename'] != 'config.yaml' or ref['model'] != expected_model):
raise ValueError('venus_config_ref repository/file/model identity mismatch')
if not isinstance(ref['revision'],str) or not re.fullmatch(r'[0-9a-f]{40}', ref['revision']):
raise ValueError('venus_config_ref requires a fixed full commit SHA')
if not isinstance(ref['sha256'],str) or not re.fullmatch(r'[0-9a-f]{64}', ref['sha256']):
raise ValueError('venus_config_ref requires a SHA-256 digest')
if config.get('model_type') != expected_types[expected_model]:
raise ValueError('Child configuration model_type mismatch')
raw = None
path = Path(model_path)
subfolder = options.get('subfolder', '')
candidates = []
if path.is_dir():
model_dir = path / subfolder if subfolder else path
candidates.append(model_dir.parent / 'config.yaml')
elif path.is_file():
candidates.append(path.parent.parent / 'config.yaml')
for candidate in candidates:
if candidate.is_file():
value = candidate.read_bytes()
if hashlib.sha256(value).hexdigest() == ref['sha256']:
raw = value
break
if raw is None:
from huggingface_hub import hf_hub_download, try_to_load_from_cache
from transformers.utils.hub import TRANSFORMERS_CACHE
cache_dir = options.get('cache_dir') or TRANSFORMERS_CACHE
cached = try_to_load_from_cache(ref['repo_id'],ref['filename'],
revision=ref['revision'],repo_type='model',cache_dir=cache_dir)
if isinstance(cached,str):
raw = Path(cached).read_bytes()
else:
download_options = {k: options[k] for k in
('token','local_files_only','force_download','user_agent','proxies') if k in options}
download_options['cache_dir'] = cache_dir
try:
resolved = hf_hub_download(repo_id=ref['repo_id'],repo_type='model',
filename=ref['filename'],revision=ref['revision'],**download_options)
except OSError as error:
raise OSError('Required pinned Realtime-Venus config.yaml is unavailable locally/in cache; '
'download that revision while online or place its verified YAML beside the model directory.') from error
raw = Path(resolved).read_bytes()
if hashlib.sha256(raw).hexdigest() != ref['sha256']:
raise ValueError('Pinned Realtime-Venus config.yaml SHA-256 mismatch')
import yaml
document = yaml.safe_load(raw)
if not isinstance(document,dict) or document.get('schema_version') != 3 or document.get('name') != 'Realtime-Venus':
raise ValueError('Invalid Realtime-Venus root YAML schema/name')
models = document.get('models')
entry = models.get(expected_model) if isinstance(models,dict) else None
expected_path = 'Realtime-Venus-Audio' if expected_model == 'audio' else 'Realtime-Venus-Omni'
if not isinstance(entry,dict) or entry.get('path') != expected_path:
raise ValueError('Root YAML model path mismatch')
full = entry.get('hf_config')
if not isinstance(full,dict) or 'venus_config_ref' in full or full.get('model_type') != expected_types[expected_model]:
raise ValueError('Root YAML hf_config model identity mismatch')
if full.get('architectures') != config.get('architectures'):
raise ValueError('Root YAML architecture mismatch')
for field in ('audio_config','vision_config','tts_config'):
if not isinstance(full.get(field),dict):
raise ValueError('Incomplete root hf_config: ' + field)
def auto_map_identity(mapping):
if not isinstance(mapping,dict):
raise ValueError('Invalid auto_map')
return {k:v.split('--')[-1] for k,v in mapping.items()}
if auto_map_identity(full.get('auto_map')) != auto_map_identity(config.get('auto_map')):
raise ValueError('Root YAML AutoClass identity mismatch')
expanded = copy.deepcopy(full)
# Keep the current model snapshot's code/weights revision, not the YAML's.
if '_commit_hash' in config:
expanded['_commit_hash'] = config['_commit_hash']
expanded['auto_map'] = copy.deepcopy(config['auto_map'])
expanded['_name_or_path'] = str(model_path)
return expanded