Backup-bdg commited on
Commit
94abce9
·
verified ·
1 Parent(s): 50b355c

Update model weights after training (epoch 1, loss 12.3928)

Browse files
audio_decoder.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:94c9d5ccf3b5202d55b05a0ff259a1d215e67e53ac8097d0ebe8f4cda6487435
3
  size 1458415836
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05d6cdd45c2610d4f71bb002bf6c1218abb0cb6b088fcaf8e003c158fd729aed
3
  size 1458415836
audio_encoder.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b8b26dc5ffc26af341bffebf3751d0b8cfda76b57f5633c051fad23de3779c6d
3
  size 466150140
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:187509c933f459db10838c955ccfde7a0d038579f35d3c969e6fabdeb62750da
3
  size 466150140
audio_projector.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:52e4d3e69047791d76dc7a5f84bb4de7af037561375fdfa7434aed82c22f9e2e
3
  size 2099352
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6971e95b46f2903c27435c6fd19d5674cb628c4157f7c800d064da3f3712afae
3
  size 2099352
cross_attention.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:aa8904527e31e2a2b52b1390dbcdedfb1b97dd9ded757711150b7ce78caa7cc4
3
  size 174191400
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09fa51e9cffd51c0472acd68d07f1da6f34d5e73446a37438762cbd761e006bd
3
  size 174191400
generator.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:985420487363f55f0e18139de710f2836189141cbe434b2e169fc7c803ac5393
3
  size 629440508
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cef683d2d47c6e2ee65619df52269080c255df156a7c253f9d4e337eca19ac8e
3
  size 629440508
llm.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:917395d8d9856d22a4913f8e3f18143cf9a038454432f79ad78644fc3e4ec485
3
  size 1506831304
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ede34b9a94be9b3888eb74c50b17ac192987ca36ea07e697d4979bfd3a578ff
3
  size 1506831304
modality_markers.safetensors CHANGED
Binary files a/modality_markers.safetensors and b/modality_markers.safetensors differ
 
modeling_xoron.py CHANGED
@@ -420,6 +420,7 @@ def count_lora_parameters(model: nn.Module) -> Tuple[int, int, float]:
420
  def flash_attention_available() -> bool:
421
  """Check if Flash Attention (via SDPA) is available."""
422
  try:
 
423
  return True
424
  except ImportError:
425
  return False
@@ -2549,6 +2550,7 @@ class VisionEncoder(nn.Module):
2549
  def _init_siglip(self, model_name: str, freeze: bool):
2550
  """Initialize SigLIP 2 vision encoder."""
2551
  try:
 
2552
 
2553
  self.vision_model = SiglipVisionModel.from_pretrained(model_name)
2554
  self.image_processor = SiglipImageProcessor.from_pretrained(model_name)
@@ -2574,6 +2576,7 @@ class VisionEncoder(nn.Module):
2574
 
2575
  def _init_clip(self, model_name: str, freeze: bool):
2576
  """Initialize CLIP vision encoder (legacy support)."""
 
2577
 
2578
  self.vision_model = CLIPVisionModel.from_pretrained(model_name)
2579
  self.image_processor = CLIPImageProcessor.from_pretrained(model_name)
@@ -5180,6 +5183,7 @@ class AudioEncoder(nn.Module):
5180
 
5181
  # Conformer blocks with optional gradient checkpointing
5182
  if self.gradient_checkpointing and self.training:
 
5183
  for block in self.conformer_blocks:
5184
  # Use checkpoint to save memory during training
5185
  def create_custom_forward(module):
@@ -5739,6 +5743,7 @@ class AudioDecoder(nn.Module):
5739
 
5740
  # Encoder with optional gradient checkpointing
5741
  if self.gradient_checkpointing and self.training:
 
5742
  for block in self.encoder_blocks:
5743
  def create_custom_forward(module):
5744
  def custom_forward(*inputs):
@@ -5789,6 +5794,7 @@ class AudioDecoder(nn.Module):
5789
 
5790
  # Decoder with optional gradient checkpointing
5791
  if self.gradient_checkpointing and self.training:
 
5792
  for block in self.decoder_blocks:
5793
  def create_custom_forward(module):
5794
  def custom_forward(*inputs):
@@ -9870,6 +9876,7 @@ class XoronMultimodalModel(nn.Module):
9870
  Args:
9871
  path: Directory to save the files
9872
  """
 
9873
 
9874
  # Get the root directory of the Xoron project
9875
  current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -9895,6 +9902,7 @@ class XoronMultimodalModel(nn.Module):
9895
  This creates a single file with ALL model code embedded, removing internal
9896
  imports so it works standalone on HuggingFace without the full package.
9897
  """
 
9898
 
9899
  # Component files to combine (in dependency order)
9900
  component_files = [
@@ -9926,13 +9934,18 @@ class XoronMultimodalModel(nn.Module):
9926
  return True
9927
  return False
9928
 
9929
- def is_external_import(line):
9930
- line = line.strip()
9931
- return (line.startswith("import ") or
9932
- (line.startswith("from ") and not is_internal_import(line)))
 
 
 
 
 
9933
 
9934
  def extract_code_body(content):
9935
- """Extract code body, removing module docstring and imports."""
9936
  lines = content.split('\n')
9937
  code_lines = []
9938
  i = 0
@@ -9965,21 +9978,22 @@ class XoronMultimodalModel(nn.Module):
9965
  if not code_lines and not stripped:
9966
  continue
9967
 
9968
- # Handle multi-line imports - skip continuation lines
9969
  if in_multiline_import:
9970
  if ')' in stripped:
9971
  in_multiline_import = False
9972
  continue
9973
 
9974
- # Check if this starts a multi-line import that should be skipped
9975
- if is_internal_import(line) or is_external_import(line):
 
9976
  # Check if it's a multi-line import (has opening paren but no closing)
9977
  if '(' in stripped and ')' not in stripped:
9978
  in_multiline_import = True
9979
  continue
9980
 
9981
- # Skip logger setup lines
9982
- if stripped.startswith("logger = logging.getLogger"):
9983
  continue
9984
 
9985
  code_lines.append(line)
@@ -10002,6 +10016,7 @@ to enable loading via AutoModel with trust_remote_code=True WITHOUT requiring
10002
  the full Xoron-Dev package to be installed.
10003
 
10004
  Usage:
 
10005
  config = AutoConfig.from_pretrained("your-repo/xoron-model", trust_remote_code=True)
10006
  model = AutoModel.from_pretrained("your-repo/xoron-model", trust_remote_code=True)
10007
  """
@@ -10009,18 +10024,25 @@ Usage:
10009
 
10010
 
10011
  try:
 
10012
  except ImportError:
10013
  save_file, load_file = None, None
10014
 
10015
 
10016
  try:
 
 
 
 
10017
  except ImportError:
10018
  LlamaAttention = LlamaDecoderLayer = LlamaRMSNorm = LlamaMLP = None
10019
  LlamaRotaryEmbedding = apply_rotary_pos_emb = repeat_kv = None
10020
 
10021
  # Import configuration
10022
  try:
 
10023
  except ImportError:
 
10024
 
10025
 
10026
  '''
@@ -10160,6 +10182,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
10160
  Args:
10161
  path: Directory to save the model
10162
  """
 
10163
 
10164
  state_dict = self.state_dict()
10165
 
@@ -10184,6 +10207,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
10184
  Args:
10185
  path: Directory to save component files
10186
  """
 
10187
 
10188
  os.makedirs(path, exist_ok=True)
10189
 
@@ -10299,6 +10323,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
10299
  path: Directory to save shards
10300
  max_shard_size: Maximum bytes per shard
10301
  """
 
10302
 
10303
  state_dict = self.state_dict()
10304
 
@@ -10393,6 +10418,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
10393
  Args:
10394
  path: Directory to save component files
10395
  """
 
10396
 
10397
  os.makedirs(path, exist_ok=True)
10398
 
@@ -10518,6 +10544,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
10518
  Returns:
10519
  Loaded XoronMultimodalModel instance
10520
  """
 
10521
 
10522
  print(f"\n📂 Loading model from {path}...")
10523
 
@@ -10620,6 +10647,7 @@ XoronForCausalLM.register_for_auto_class("AutoModelForCausalLM")
10620
  path: Directory containing component files
10621
  strict: If True, require exact match; if False, allow partial loading
10622
  """
 
10623
 
10624
  # Component mapping from file to model attribute
10625
  component_map = {
 
420
  def flash_attention_available() -> bool:
421
  """Check if Flash Attention (via SDPA) is available."""
422
  try:
423
+ from torch.nn.functional import scaled_dot_product_attention
424
  return True
425
  except ImportError:
426
  return False
 
2550
  def _init_siglip(self, model_name: str, freeze: bool):
2551
  """Initialize SigLIP 2 vision encoder."""
2552
  try:
2553
+ from transformers import SiglipVisionModel, SiglipImageProcessor
2554
 
2555
  self.vision_model = SiglipVisionModel.from_pretrained(model_name)
2556
  self.image_processor = SiglipImageProcessor.from_pretrained(model_name)
 
2576
 
2577
  def _init_clip(self, model_name: str, freeze: bool):
2578
  """Initialize CLIP vision encoder (legacy support)."""
2579
+ from transformers import CLIPVisionModel, CLIPImageProcessor
2580
 
2581
  self.vision_model = CLIPVisionModel.from_pretrained(model_name)
2582
  self.image_processor = CLIPImageProcessor.from_pretrained(model_name)
 
5183
 
5184
  # Conformer blocks with optional gradient checkpointing
5185
  if self.gradient_checkpointing and self.training:
5186
+ from torch.utils.checkpoint import checkpoint
5187
  for block in self.conformer_blocks:
5188
  # Use checkpoint to save memory during training
5189
  def create_custom_forward(module):
 
5743
 
5744
  # Encoder with optional gradient checkpointing
5745
  if self.gradient_checkpointing and self.training:
5746
+ from torch.utils.checkpoint import checkpoint
5747
  for block in self.encoder_blocks:
5748
  def create_custom_forward(module):
5749
  def custom_forward(*inputs):
 
5794
 
5795
  # Decoder with optional gradient checkpointing
5796
  if self.gradient_checkpointing and self.training:
5797
+ from torch.utils.checkpoint import checkpoint
5798
  for block in self.decoder_blocks:
5799
  def create_custom_forward(module):
5800
  def custom_forward(*inputs):
 
9876
  Args:
9877
  path: Directory to save the files
9878
  """
9879
+ import shutil
9880
 
9881
  # Get the root directory of the Xoron project
9882
  current_dir = os.path.dirname(os.path.abspath(__file__))
 
9902
  This creates a single file with ALL model code embedded, removing internal
9903
  imports so it works standalone on HuggingFace without the full package.
9904
  """
9905
+ import re
9906
 
9907
  # Component files to combine (in dependency order)
9908
  component_files = [
 
9934
  return True
9935
  return False
9936
 
9937
+ def is_module_level_import(line):
9938
+ """Check if this is a module-level import (no indentation)."""
9939
+ stripped = line.strip()
9940
+ # Only consider it a module-level import if:
9941
+ # 1. Line starts with no indentation (module level)
9942
+ # 2. Line starts with 'import' or 'from'
9943
+ if line and not line[0].isspace():
9944
+ return (stripped.startswith("import ") or stripped.startswith("from "))
9945
+ return False
9946
 
9947
  def extract_code_body(content):
9948
+ """Extract code body, removing module docstring and module-level imports only."""
9949
  lines = content.split('\n')
9950
  code_lines = []
9951
  i = 0
 
9978
  if not code_lines and not stripped:
9979
  continue
9980
 
9981
+ # Handle multi-line module-level imports - skip continuation lines
9982
  if in_multiline_import:
9983
  if ')' in stripped:
9984
  in_multiline_import = False
9985
  continue
9986
 
9987
+ # Only skip MODULE-LEVEL imports (no indentation)
9988
+ # Keep inline/function-level imports like: from transformers import X
9989
+ if is_module_level_import(line):
9990
  # Check if it's a multi-line import (has opening paren but no closing)
9991
  if '(' in stripped and ')' not in stripped:
9992
  in_multiline_import = True
9993
  continue
9994
 
9995
+ # Skip logger setup lines at module level only
9996
+ if stripped.startswith("logger = logging.getLogger") and not line[0].isspace():
9997
  continue
9998
 
9999
  code_lines.append(line)
 
10016
  the full Xoron-Dev package to be installed.
10017
 
10018
  Usage:
10019
+ from transformers import AutoModel, AutoConfig
10020
  config = AutoConfig.from_pretrained("your-repo/xoron-model", trust_remote_code=True)
10021
  model = AutoModel.from_pretrained("your-repo/xoron-model", trust_remote_code=True)
10022
  """
 
10024
 
10025
 
10026
  try:
10027
+ from safetensors.torch import save_file, load_file
10028
  except ImportError:
10029
  save_file, load_file = None, None
10030
 
10031
 
10032
  try:
10033
+ from transformers.models.llama.modeling_llama import (
10034
+ LlamaAttention, LlamaDecoderLayer, LlamaRMSNorm, LlamaMLP,
10035
+ LlamaRotaryEmbedding, apply_rotary_pos_emb, repeat_kv
10036
+ )
10037
  except ImportError:
10038
  LlamaAttention = LlamaDecoderLayer = LlamaRMSNorm = LlamaMLP = None
10039
  LlamaRotaryEmbedding = apply_rotary_pos_emb = repeat_kv = None
10040
 
10041
  # Import configuration
10042
  try:
10043
+ from .configuration_xoron import XoronConfig
10044
  except ImportError:
10045
+ from configuration_xoron import XoronConfig
10046
 
10047
 
10048
  '''
 
10182
  Args:
10183
  path: Directory to save the model
10184
  """
10185
+ from safetensors.torch import save_file
10186
 
10187
  state_dict = self.state_dict()
10188
 
 
10207
  Args:
10208
  path: Directory to save component files
10209
  """
10210
+ from safetensors.torch import save_file
10211
 
10212
  os.makedirs(path, exist_ok=True)
10213
 
 
10323
  path: Directory to save shards
10324
  max_shard_size: Maximum bytes per shard
10325
  """
10326
+ from safetensors.torch import save_file
10327
 
10328
  state_dict = self.state_dict()
10329
 
 
10418
  Args:
10419
  path: Directory to save component files
10420
  """
10421
+ from safetensors.torch import save_file
10422
 
10423
  os.makedirs(path, exist_ok=True)
10424
 
 
10544
  Returns:
10545
  Loaded XoronMultimodalModel instance
10546
  """
10547
+ from safetensors import safe_open
10548
 
10549
  print(f"\n📂 Loading model from {path}...")
10550
 
 
10647
  path: Directory containing component files
10648
  strict: If True, require exact match; if False, allow partial loading
10649
  """
10650
+ from safetensors import safe_open
10651
 
10652
  # Component mapping from file to model attribute
10653
  component_map = {
projector.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:09b66cb42848d8091b0242839d5ce7c15eb8cf300a0da5b7112dc168b5be9de8
3
  size 52880664
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c28b5916bb6a53e9ebc817a1e00532a985ff0d94a552fb4cd8ad9d0958a8928b
3
  size 52880664
streaming_state.json CHANGED
@@ -3,12 +3,12 @@
3
  "unique_samples": 1,
4
  "total_yields": 2,
5
  "dataset_positions": {
6
- "NewYorker": 1
7
  },
8
  "modality_positions": {
9
  "text": {},
10
  "image": {
11
- "NewYorker": 1
12
  },
13
  "video": {},
14
  "audio": {}
 
3
  "unique_samples": 1,
4
  "total_yields": 2,
5
  "dataset_positions": {
6
+ "WebSight": 1
7
  },
8
  "modality_positions": {
9
  "text": {},
10
  "image": {
11
+ "WebSight": 1
12
  },
13
  "video": {},
14
  "audio": {}
trainer_state.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "best_model_checkpoint": "/kaggle/working/xoron-final",
3
- "best_metric": 12.83968734741211,
4
  "epoch": 1,
5
  "epochs_completed": 1,
6
  "global_step": 0,
 
1
  {
2
  "best_model_checkpoint": "/kaggle/working/xoron-final",
3
+ "best_metric": 12.392779350280762,
4
  "epoch": 1,
5
  "epochs_completed": 1,
6
  "global_step": 0,
training_state.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e854ff708185d68e08462f8de36816e1ee9d51515b7ae6551f14333ba1ff8c66
3
  size 5143
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfe96eea50ca0d57808883eae325d7f8cb995a440f4e5b6f350fecf7e2c8a97b
3
  size 5143
video_encoder.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:26ef05c4987b904dcff3114946c83e8331db289e4e135fd9fdc917d133a2c05a
3
  size 1923089112
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8201d34db9c136636576e482c29124a05faf904a43b6e928c1ddf854a6f8e3b
3
  size 1923089112
video_generator.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b60ea2266cbe650df7dd9f10d3d705148b1eb6e84adcad2226a38d9f9a241ed8
3
  size 61574134
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c36fae8183f3b076c294749e84a3ad78735ba2baff05800cf9287908e168e5c4
3
  size 61574134
vision_encoder.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8622314b7c58a633e2a32028742516f3d59d21010bb11403b7d261d8dd066150
3
  size 1000535480
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f020238f58c66b573d8d521d99480fd6dc685799e902fcbfd52789702205eab0
3
  size 1000535480
waveform_decoder.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7fafc4dbb6271b5c66ddd57054d4aa50c06bea14b7a30952ebbefc2168a5bad7
3
  size 34681076
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf34349303d6dbabc1d729311a57cf09faa1d6031889873c91b5e338356993b0
3
  size 34681076