Spaces:
Sleeping
Sleeping
File size: 3,114 Bytes
211bdc2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 |
import os
import subprocess
import sys
import zipfile
import urllib.request
from pathlib import Path
def download_file(url, filename):
"""Download a file with progress"""
print(f"Downloading {filename}...")
urllib.request.urlretrieve(url, filename)
print(f"✅ Downloaded {filename}")
def setup_indiclid():
"""Setup IndicLID manually"""
print("=== Setting up IndicLID ===")
# Create directory structure
os.makedirs("ai4bharat", exist_ok=True)
os.makedirs("models", exist_ok=True)
# Download IndicLID source code
print("Downloading IndicLID source...")
subprocess.run([
"git", "clone", "--depth", "1",
"https://github.com/AI4Bharat/IndicLID.git",
"temp_indiclid"
], check=True)
# Copy necessary files
import shutil
source_file = "temp_indiclid/Inference/ai4bharat/IndicLID.py"
if os.path.exists(source_file):
shutil.copy2(source_file, "ai4bharat/IndicLID.py")
print("✅ Copied IndicLID.py")
# Create __init__.py
with open("ai4bharat/__init__.py", "w") as f:
f.write("")
# Clean up
shutil.rmtree("temp_indiclid", ignore_errors=True)
# Download model files
model_urls = [
"https://github.com/AI4Bharat/IndicLID/releases/download/v1.0/indiclid-bert.zip",
"https://github.com/AI4Bharat/IndicLID/releases/download/v1.0/indiclid-ftn.zip",
"https://github.com/AI4Bharat/IndicLID/releases/download/v1.0/indiclid-ftr.zip"
]
os.chdir("models")
for url in model_urls:
filename = url.split("/")[-1]
try:
download_file(url, filename)
# Extract zip file
print(f"Extracting {filename}...")
with zipfile.ZipFile(filename, 'r') as zip_ref:
zip_ref.extractall('.')
# Remove zip file
os.remove(filename)
print(f"✅ Extracted and cleaned {filename}")
except Exception as e:
print(f"❌ Error with {filename}: {e}")
os.chdir("..")
print("✅ IndicLID setup complete!")
def patch_indiclid():
"""Apply necessary patches to IndicLID"""
indiclid_file = "ai4bharat/IndicLID.py"
if not os.path.exists(indiclid_file):
print("❌ IndicLID.py not found!")
return
print("Applying patches to IndicLID.py...")
# Read the file
with open(indiclid_file, "r") as f:
content = f.read()
# Apply patches
content = content.replace(
"torch.load(self.IndicLID_BERT_path, map_location = self.device)",
"torch.load(self.IndicLID_BERT_path, map_location=self.device, weights_only=False)"
)
# Write back
with open(indiclid_file, "w") as f:
f.write(content)
print("✅ Patches applied successfully!")
if __name__ == "__main__":
try:
setup_indiclid()
patch_indiclid()
print("\n🎉 Setup completed successfully!")
except Exception as e:
print(f"\n❌ Setup failed: {e}")
sys.exit(1)
|