Santosh-bold commited on
Commit
71f8cea
·
verified ·
1 Parent(s): 96b75ae

Upload model.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. model.py +80 -0
model.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import mlx.core as mx
2
+ import mlx.nn as nn
3
+ from mlx_lm.models.base import BaseModelArgs
4
+ from mlx_lm.utils import load
5
+
6
+ from subsampling import ConvSubsampling
7
+ from modules import ConformerLayer
8
+ from attention import RelPositionalEncoding
9
+
10
+ class FastConformerEncoder(nn.Module):
11
+ def __init__(
12
+ self,
13
+ feat_in: int = 80,
14
+ n_layers: int = 18,
15
+ d_model: int = 512,
16
+ ff_expansion_factor: int = 4,
17
+ n_heads: int = 8,
18
+ conv_kernel_size: int = 9,
19
+ dropout: float = 0.1,
20
+ ):
21
+ super().__init__()
22
+ self.d_model = d_model
23
+ # FastConformer uses 8x downsampling via striding
24
+ self.pre_encode = ConvSubsampling(
25
+ subsampling='dw_striding',
26
+ subsampling_factor=8,
27
+ feat_in=feat_in,
28
+ feat_out=d_model,
29
+ conv_channels=256
30
+ )
31
+
32
+ self.pos_enc = RelPositionalEncoding(d_model=d_model, max_len=5000)
33
+
34
+ d_ff = d_model * ff_expansion_factor
35
+ self.layers = [
36
+ ConformerLayer(
37
+ d_model=d_model,
38
+ d_ff=d_ff,
39
+ n_heads=n_heads,
40
+ conv_kernel_size=conv_kernel_size,
41
+ dropout=dropout
42
+ ) for _ in range(n_layers)
43
+ ]
44
+
45
+ def __call__(self, x, lengths=None):
46
+ # x: (batch, time, feat_in)
47
+ if lengths is None:
48
+ lengths = mx.array([x.shape[1]] * x.shape[0])
49
+
50
+ x, lengths = self.pre_encode(x, lengths)
51
+
52
+ # Add positional encoding
53
+ x, pos_emb = self.pos_enc(x)
54
+
55
+ for layer in self.layers:
56
+ x = layer(x, pos_emb=pos_emb)
57
+
58
+ return x, lengths
59
+
60
+ class CanaryModel(nn.Module):
61
+ """
62
+ Hybrid ASR-LLM Model connecting FastConformer to Qwen via a linear projection.
63
+ """
64
+ def __init__(self, encoder: FastConformerEncoder, llm_dim: int):
65
+ super().__init__()
66
+ self.encoder = encoder
67
+
68
+ # Audio to Text projection (LoRA or standard linear depending on NeMo config)
69
+ self.audio_encoder_proj = nn.Linear(1024, llm_dim)
70
+
71
+ def encode_audio(self, audio_features, lengths=None):
72
+ """
73
+ Passes audio features through the encoder and projects them to the LLM embedding space.
74
+ """
75
+ encoded_audio, lengths = self.encoder(audio_features, lengths)
76
+ audio_embeds = self.audio_encoder_proj(encoded_audio)
77
+ return audio_embeds, lengths
78
+
79
+ # Note: Text generation logic will be handled externally by feeding
80
+ # the audio_embeds directly into the mlx_lm Qwen decoder prompt embedding!