setup_models.py

import os
import subprocess
import sys
import zipfile
import urllib.request
from pathlib import Path

def download_file(url, filename):
    """Download a file with progress"""
    print(f"Downloading {filename}...")
    urllib.request.urlretrieve(url, filename)
    print(f"✅ Downloaded {filename}")

def setup_indiclid():
    """Setup IndicLID manually"""
    print("=== Setting up IndicLID ===")
    
    # Create directory structure
    os.makedirs("ai4bharat", exist_ok=True)
    os.makedirs("models", exist_ok=True)
    
    # Download IndicLID source code
    print("Downloading IndicLID source...")
    subprocess.run([
        "git", "clone", "--depth", "1", 
        "https://github.com/AI4Bharat/IndicLID.git", 
        "temp_indiclid"
    ], check=True)
    
    # Copy necessary files
    import shutil
    source_file = "temp_indiclid/Inference/ai4bharat/IndicLID.py"
    if os.path.exists(source_file):
        shutil.copy2(source_file, "ai4bharat/IndicLID.py")
        print("✅ Copied IndicLID.py")
    
    # Create __init__.py
    with open("ai4bharat/__init__.py", "w") as f:
        f.write("")
    
    # Clean up
    shutil.rmtree("temp_indiclid", ignore_errors=True)
    
    # Download model files
    model_urls = [
        "https://github.com/AI4Bharat/IndicLID/releases/download/v1.0/indiclid-bert.zip",
        "https://github.com/AI4Bharat/IndicLID/releases/download/v1.0/indiclid-ftn.zip",
        "https://github.com/AI4Bharat/IndicLID/releases/download/v1.0/indiclid-ftr.zip"
    ]
    
    os.chdir("models")
    
    for url in model_urls:
        filename = url.split("/")[-1]
        try:
            download_file(url, filename)
            
            # Extract zip file
            print(f"Extracting {filename}...")
            with zipfile.ZipFile(filename, 'r') as zip_ref:
                zip_ref.extractall('.')
            
            # Remove zip file
            os.remove(filename)
            print(f"✅ Extracted and cleaned {filename}")
            
        except Exception as e:
            print(f"❌ Error with {filename}: {e}")
    
    os.chdir("..")
    print("✅ IndicLID setup complete!")

def patch_indiclid():
    """Apply necessary patches to IndicLID"""
    indiclid_file = "ai4bharat/IndicLID.py"
    if not os.path.exists(indiclid_file):
        print("❌ IndicLID.py not found!")
        return
    
    print("Applying patches to IndicLID.py...")
    
    # Read the file
    with open(indiclid_file, "r") as f:
        content = f.read()
    
    # Apply patches
    content = content.replace(
        "torch.load(self.IndicLID_BERT_path, map_location = self.device)",
        "torch.load(self.IndicLID_BERT_path, map_location=self.device, weights_only=False)"
    )
    
    # Write back
    with open(indiclid_file, "w") as f:
        f.write(content)
    
    print("✅ Patches applied successfully!")

if __name__ == "__main__":
    try:
        setup_indiclid()
        patch_indiclid()
        print("\n🎉 Setup completed successfully!")
    except Exception as e:
        print(f"\n❌ Setup failed: {e}")
        sys.exit(1)