# #!/usr/bin/env python3 # """ # Local setup script for Manufacturing RAG Agent without Docker. # This script helps set up the RAG system using Qdrant Cloud or local alternatives. # """ # import os # import sys # import subprocess # from pathlib import Path # import requests # import time # def check_python_version(): # """Check if Python version is compatible.""" # if sys.version_info < (3, 8): # print("โŒ Python 3.8+ is required") # return False # print(f"โœ… Python {sys.version_info.major}.{sys.version_info.minor} detected") # return True # def check_dependencies(): # """Check if required dependencies are installed.""" # print("\n๐Ÿ” Checking dependencies...") # # Check pip packages # required_packages = [ # 'streamlit', 'qdrant-client', 'groq', 'requests', # 'pandas', 'plotly', 'PyMuPDF', 'openpyxl', 'pytesseract', 'Pillow' # ] # missing_packages = [] # for package in required_packages: # try: # __import__(package.replace('-', '_').lower()) # print(f"โœ… {package}") # except ImportError: # print(f"โŒ {package}") # missing_packages.append(package) # if missing_packages: # print(f"\n๐Ÿ“ฆ Installing missing packages: {', '.join(missing_packages)}") # try: # subprocess.check_call([ # sys.executable, '-m', 'pip', 'install' # ] + missing_packages) # print("โœ… All packages installed successfully") # except subprocess.CalledProcessError: # print("โŒ Failed to install packages") # return False # return True # def check_tesseract(): # """Check if Tesseract OCR is installed.""" # print("\n๐Ÿ” Checking Tesseract OCR...") # try: # result = subprocess.run(['tesseract', '--version'], # capture_output=True, text=True) # if result.returncode == 0: # version = result.stdout.split('\n')[0] # print(f"โœ… {version}") # return True # except FileNotFoundError: # pass # print("โŒ Tesseract OCR not found") # print("๐Ÿ“ฅ Install with: brew install tesseract") # return False # def setup_environment(): # """Set up environment variables.""" # print("\nโš™๏ธ Setting up environment...") # env_file = Path('.env') # env_example = Path('.env.example') # if not env_file.exists() and env_example.exists(): # # Copy example file # with open(env_example, 'r') as f: # content = f.read() # with open(env_file, 'w') as f: # f.write(content) # print("โœ… Created .env file from template") # print("๐Ÿ“ Please edit .env file with your API keys:") # print(" - GROQ_API_KEY (get from https://console.groq.com/)") # print(" - SILICONFLOW_API_KEY (get from https://siliconflow.cn/)") # print(" - QDRANT_URL and QDRANT_API_KEY (if using Qdrant Cloud)") # return False # elif env_file.exists(): # print("โœ… .env file exists") # return True # else: # print("โŒ No .env.example file found") # return False # def test_qdrant_connection(): # """Test Qdrant connection.""" # print("\n๐Ÿ” Testing Qdrant connection...") # # Try local Qdrant first # try: # response = requests.get('http://localhost:6333/health', timeout=5) # if response.status_code == 200: # print("โœ… Local Qdrant is running") # return True # except requests.exceptions.RequestException: # pass # # Try Qdrant Cloud if configured # qdrant_url = os.getenv('QDRANT_URL') # qdrant_key = os.getenv('QDRANT_API_KEY') # if qdrant_url and qdrant_key: # try: # headers = {'api-key': qdrant_key} if qdrant_key else {} # response = requests.get(f"{qdrant_url}/health", headers=headers, timeout=10) # if response.status_code == 200: # print("โœ… Qdrant Cloud connection successful") # return True # except requests.exceptions.RequestException as e: # print(f"โŒ Qdrant Cloud connection failed: {e}") # print("โŒ No Qdrant connection available") # print("๐Ÿ’ก Options:") # print(" 1. Start local Qdrant: docker run -p 6333:6333 qdrant/qdrant") # print(" 2. Use Qdrant Cloud: https://cloud.qdrant.io/") # print(" 3. Download Qdrant binary: https://github.com/qdrant/qdrant/releases") # return False # def create_data_directories(): # """Create necessary data directories.""" # print("\n๐Ÿ“ Creating data directories...") # directories = [ # 'data', # 'data/documents', # 'logs', # 'results' # ] # for directory in directories: # Path(directory).mkdir(parents=True, exist_ok=True) # print(f"โœ… {directory}/") # return True # def test_api_keys(): # """Test API key configuration.""" # print("\n๐Ÿ”‘ Testing API keys...") # # Load environment variables # from dotenv import load_dotenv # load_dotenv() # # Test Groq API # groq_key = os.getenv('GROQ_API_KEY') # if groq_key: # try: # headers = {'Authorization': f'Bearer {groq_key}'} # response = requests.get('https://api.groq.com/openai/v1/models', # headers=headers, timeout=10) # if response.status_code == 200: # print("โœ… Groq API key is valid") # else: # print(f"โŒ Groq API key test failed: {response.status_code}") # except requests.exceptions.RequestException as e: # print(f"โŒ Groq API connection failed: {e}") # else: # print("โŒ GROQ_API_KEY not set") # # Test Silicon Flow API # sf_key = os.getenv('SILICONFLOW_API_KEY') # if sf_key: # print("โœ… Silicon Flow API key is set (cannot test without making API call)") # else: # print("โŒ SILICONFLOW_API_KEY not set") # return groq_key and sf_key # def main(): # """Main setup function.""" # print("๐Ÿญ Manufacturing RAG Agent Setup") # print("=" * 40) # # Check Python version # if not check_python_version(): # sys.exit(1) # # Check dependencies # if not check_dependencies(): # print("\nโŒ Dependency check failed") # sys.exit(1) # # Check Tesseract # tesseract_ok = check_tesseract() # # Setup environment # env_setup = setup_environment() # # Create directories # create_data_directories() # if env_setup: # # Test API keys # api_keys_ok = test_api_keys() # # Test Qdrant # qdrant_ok = test_qdrant_connection() # print("\n" + "=" * 40) # print("๐Ÿ“‹ Setup Summary:") # print(f"โœ… Python: OK") # print(f"โœ… Dependencies: OK") # print(f"{'โœ…' if tesseract_ok else 'โŒ'} Tesseract OCR: {'OK' if tesseract_ok else 'Missing'}") # print(f"{'โœ…' if api_keys_ok else 'โŒ'} API Keys: {'OK' if api_keys_ok else 'Missing'}") # print(f"{'โœ…' if qdrant_ok else 'โŒ'} Qdrant: {'OK' if qdrant_ok else 'Not available'}") # if tesseract_ok and api_keys_ok and qdrant_ok: # print("\n๐ŸŽ‰ Setup complete! You can now run:") # print(" python launch_rag_demo.py") # else: # print("\nโš ๏ธ Setup incomplete. Please address the issues above.") # if not tesseract_ok: # print(" Install Tesseract: brew install tesseract") # if not api_keys_ok: # print(" Configure API keys in .env file") # if not qdrant_ok: # print(" Set up Qdrant (see options above)") # else: # print("\n๐Ÿ“ Please configure your .env file with API keys, then run this script again.") # if __name__ == "__main__": # main() #!/usr/bin/env python3 """ Quick fix script for Manufacturing RAG Agent issues """ import os import sys from dotenv import load_dotenv from qdrant_client import QdrantClient from qdrant_client.http import models import requests load_dotenv() def check_environment(): """Check all environment variables.""" print("๐Ÿ” Checking Environment Variables") print("=" * 40) required_vars = { 'GROQ_API_KEY': 'Groq LLM API', 'SILICONFLOW_API_KEY': 'SiliconFlow Embedding API', 'QDRANT_URL': 'Qdrant Vector Database URL', 'QDRANT_API_KEY': 'Qdrant API Key' } issues = [] for var, description in required_vars.items(): value = os.getenv(var) if value: # Mask sensitive info if 'KEY' in var: display_value = f"{value[:8]}...{value[-4:]}" if len(value) > 12 else "***" else: display_value = value print(f"โœ… {var}: {display_value}") else: print(f"โŒ {var}: Not set") issues.append(f"{var} ({description})") if issues: print(f"\nโŒ Missing environment variables:") for issue in issues: print(f" - {issue}") return False return True def test_siliconflow_api(): """Test SiliconFlow API connection and get actual embedding dimensions.""" print("\n๐Ÿงช Testing SiliconFlow API") print("=" * 30) api_key = os.getenv('SILICONFLOW_API_KEY') if not api_key: print("โŒ No API key found") return None try: payload = { "model": "Qwen/Qwen3-Embedding-8B", "input": ["test embedding to check dimensions"], "encoding_format": "float" } headers = { 'Authorization': f'Bearer {api_key}', 'Content-Type': 'application/json' } print("๐Ÿ“ก Testing API connection...") response = requests.post( "https://api.siliconflow.com/v1/embeddings", json=payload, headers=headers, timeout=15 ) if response.status_code == 200: data = response.json() if 'data' in data and len(data['data']) > 0: embedding = data['data'][0]['embedding'] dimensions = len(embedding) print(f"โœ… API working! Embedding dimensions: {dimensions}") return dimensions else: print("โŒ No embedding data returned") elif response.status_code == 401: print("โŒ API Key Invalid - Please check your SILICONFLOW_API_KEY") print(" Get a new key from: https://siliconflow.com/") else: print(f"โŒ API Error: {response.status_code}") print(f"Response: {response.text}") except Exception as e: print(f"โŒ Connection error: {e}") return None def test_qdrant_connection(): """Test Qdrant connection.""" print("\n๐Ÿ—„๏ธ Testing Qdrant Connection") print("=" * 35) qdrant_api_key= os.getenv('QDRANT_API_KEY', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIn0.DHeUsIY234NwS-6cYDJec807Vdzbs1PHmBBU3_Jz9oo') # QDRANT_URL=os.getenv('QDRANT_URL', 'https://50f53cc8-bbb0-4939-8254-8f025a577222.us-west-2-0.aws.cloud.qdrant.io:6333') # qdrant_api_key = "" qdrant_url= os.getenv('QDRANT_URL', 'http://localhost:6333') if not qdrant_url: print("โŒ No Qdrant URL found") return False try: print(f"๐Ÿ”— Connecting to: {qdrant_url}") client = QdrantClient(url=qdrant_url, api_key=qdrant_api_key) # Test connection collections = client.get_collections() print(f"โœ… Connected! Found {len(collections.collections)} collections") # Check manufacturing_docs collection collection_names = [col.name for col in collections.collections] if 'manufacturing_docs' in collection_names: collection_info = client.get_collection('manufacturing_docs') current_dim = collection_info.config.params.vectors.size points_count = collection_info.points_count print(f"๐Ÿ“‹ Collection 'manufacturing_docs' exists:") print(f" - Vector dimensions: {current_dim}") print(f" - Points count: {points_count}") return current_dim else: print("โ„น๏ธ Collection 'manufacturing_docs' doesn't exist yet") return 0 except Exception as e: print(f"โŒ Connection failed: {e}") return False def fix_qdrant_collection(correct_dimensions): """Fix the Qdrant collection with correct dimensions.""" print(f"\n๐Ÿ”ง Fixing Qdrant Collection (Dimensions: {correct_dimensions})") print("=" * 60) qdrant_url = os.getenv('QDRANT_URL') qdrant_api_key = os.getenv('QDRANT_API_KEY') collection_name = 'manufacturing_docs' try: client = QdrantClient(url=qdrant_url, api_key=qdrant_api_key) # Check current collection collections = client.get_collections() collection_names = [col.name for col in collections.collections] if collection_name in collection_names: collection_info = client.get_collection(collection_name) current_dim = collection_info.config.params.vectors.size if current_dim == correct_dimensions: print(f"โœ… Collection already has correct dimensions ({correct_dimensions})") return True print(f"๐Ÿ—‘๏ธ Deleting existing collection (wrong dimensions: {current_dim})...") client.delete_collection(collection_name) # Create new collection print(f"๐Ÿ†• Creating collection with {correct_dimensions} dimensions...") client.create_collection( collection_name=collection_name, vectors_config=models.VectorParams( size=correct_dimensions, distance=models.Distance.COSINE ) ) # Create indexes print("๐Ÿ” Creating payload indexes...") indexes = [ ("document_id", models.KeywordIndexParams()), ("chunk_id", models.KeywordIndexParams()), ("page_number", models.IntegerIndexParams()), ("worksheet_name", models.KeywordIndexParams()), ] for field_name, field_schema in indexes: try: client.create_payload_index( collection_name=collection_name, field_name=field_name, field_schema=field_schema ) except Exception as e: print(f"โš ๏ธ Index creation warning for {field_name}: {e}") print("โœ… Collection fixed successfully!") return True except Exception as e: print(f"โŒ Failed to fix collection: {e}") return False def update_gradio_demo(): """Update the Gradio demo with correct vector dimensions.""" print("\n๐Ÿ“ Updating Gradio Demo") print("=" * 25) # Check if the demo file exists demo_files = [ 'fixed_gradio_demo.py', 'fixed_gradio_file_handling.py', 'gradio_demo.py' ] demo_file = None for file in demo_files: if os.path.exists(file): demo_file = file break if not demo_file: print("โŒ No Gradio demo file found") print("Please create fixed_gradio_demo.py with the corrected code") return False try: # Read the file with open(demo_file, 'r') as f: content = f.read() # Update vector_size if "'vector_size': 1024," in content: content = content.replace("'vector_size': 1024,", "'vector_size': 4096,") print("โœ… Updated vector_size from 1024 to 4096") elif "'vector_size': 4096," in content: print("โœ… Vector size already correct (4096)") else: print("โš ๏ธ Could not find vector_size in demo file") # Write back with open(demo_file, 'w') as f: f.write(content) print(f"โœ… Updated {demo_file}") return True except Exception as e: print(f"โŒ Failed to update demo: {e}") return False def create_quick_demo(): """Create a quick working demo file.""" print("\n๐Ÿš€ Creating Quick Demo") print("=" * 22) demo_content = '''import gradio as gr import os from dotenv import load_dotenv load_dotenv() # Quick test demo def test_apis(): """Test API connections.""" results = [] # Test Groq groq_key = os.getenv('GROQ_API_KEY') results.append(f"Groq API Key: {'โœ… Set' if groq_key else 'โŒ Missing'}") # Test SiliconFlow sf_key = os.getenv('SILICONFLOW_API_KEY') results.append(f"SiliconFlow API Key: {'โœ… Set' if sf_key else 'โŒ Missing'}") # Test Qdrant qdrant_url = os.getenv('QDRANT_URL') results.append(f"Qdrant URL: {'โœ… Set' if qdrant_url else 'โŒ Missing'}") return "\\n".join(results) # Create simple interface with gr.Blocks(title="RAG System Test") as demo: gr.Markdown("# ๐Ÿงช RAG System API Test") test_btn = gr.Button("Test APIs") output = gr.Textbox(label="Results", lines=10) test_btn.click(test_apis, outputs=output) if __name__ == "__main__": demo.launch(server_name="0.0.0.0", server_port=7860) ''' with open('quick_test_demo.py', 'w') as f: f.write(demo_content) print("โœ… Created quick_test_demo.py") return True def main(): """Main fix function.""" print("๐Ÿญ Manufacturing RAG Agent - Quick Fix") print("=" * 50) # Step 1: Check environment if not check_environment(): print("\nโŒ Environment issues found. Please fix your .env file first.") return # Step 2: Test SiliconFlow API and get dimensions dimensions = test_siliconflow_api() if not dimensions: print("\nโŒ SiliconFlow API test failed. Please check your API key.") return # Step 3: Test Qdrant current_dim = test_qdrant_connection() if current_dim is False: print("\nโŒ Qdrant connection failed. Please check your Qdrant configuration.") return # Step 4: Fix Qdrant collection if needed if current_dim != dimensions: print(f"\nโš ๏ธ Collection needs fixing: {current_dim} โ†’ {dimensions}") if fix_qdrant_collection(dimensions): print("โœ… Qdrant collection fixed!") else: print("โŒ Failed to fix Qdrant collection") return # Step 5: Update demo file if update_gradio_demo(): print("โœ… Demo file updated!") # Step 6: Create quick test demo create_quick_demo() print("\n๐ŸŽ‰ All fixes applied!") print("\n๐Ÿ“‹ Next Steps:") print("1. Test APIs: python quick_test_demo.py") print("2. Run full demo: python fixed_gradio_demo.py") print("3. Upload documents and test questions") if __name__ == "__main__": main()