Add Firecrawl integration for superior web scraping

config.py

@@ -9,6 +9,7 @@ load_dotenv()
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "your_openai_key_here")
 ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "your_anthropic_key_here")
 SERPAPI_KEY = os.getenv("SERPAPI_KEY", "your_serpapi_key_here")
+FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY", "fc-08e46542bfcc4ca7a953fac4dea4237e")
 
 # LLM Configuration
 LLM_CONFIG: Dict[str, Any] = {

micro/scrape.py

@@ -4,6 +4,15 @@ import re
 from typing import Dict, Tuple, Optional
 from datetime import datetime
 
+# Try to import Firecrawl
+try:
+    from firecrawl import FirecrawlApp
+    from config import FIRECRAWL_API_KEY
+    FIRECRAWL_AVAILABLE = True and FIRECRAWL_API_KEY != "your_firecrawl_key_here"
+except ImportError:
+    FIRECRAWL_AVAILABLE = False
+    print("Warning: Firecrawl not available. Web scraping will use fallback methods.")
+
 # Try to import Selenium, but handle gracefully if not available
 try:
     from selenium import webdriver
@@ -149,6 +158,17 @@ class ScrapeMicroFunction:
     """Micro-function for web scraping with enhanced preview extraction"""
     
     def __init__(self):
+        # Initialize Firecrawl client if available
+        if FIRECRAWL_AVAILABLE:
+            try:
+                self.firecrawl_app = FirecrawlApp(api_key=FIRECRAWL_API_KEY)
+                print("✅ Firecrawl client initialized successfully")
+            except Exception as e:
+                print(f"⚠️ Firecrawl initialization failed: {e}")
+                self.firecrawl_app = None
+        else:
+            self.firecrawl_app = None
+            
         if SELENIUM_AVAILABLE:
             self.chrome_options = Options()
             self.chrome_options.add_argument('--headless')
@@ -212,8 +232,11 @@ class ScrapeMicroFunction:
     def _scrape_url(self, url: str) -> dict:
         """Scrape URL and extract both preview and full content"""
         try:
-            # Try LinkedIn-specific scraping first
-            if 'linkedin.com' in url:
+            # Try Firecrawl first if available (works for all sites including LinkedIn)
+            if self.firecrawl_app:
+                return self._scrape_with_firecrawl(url)
+            # Fallback to site-specific methods
+            elif 'linkedin.com' in url:
                 return self._scrape_linkedin(url)
             else:
                 return self._scrape_generic(url)
@@ -301,6 +324,130 @@ class ScrapeMicroFunction:
             # Fallback to Selenium
             return self._scrape_with_selenium(url)
     
+    def _scrape_with_firecrawl(self, url: str) -> dict:
+        """Firecrawl scraping - works for all sites including LinkedIn"""
+        if not self.firecrawl_app:
+            # Fallback to other methods if Firecrawl not available
+            if 'linkedin.com' in url:
+                return self._scrape_linkedin(url)
+            else:
+                return self._scrape_generic(url)
+                
+        try:
+            print(f"🔥 Using Firecrawl to scrape: {url}")
+            
+            # Use Firecrawl to scrape the URL and get LLM-ready markdown
+            scrape_result = self.firecrawl_app.scrape_url(
+                url, 
+                formats=['markdown', 'html'],
+                only_main_content=True,  # Focus on main content
+                timeout=30000
+            )
+            
+            if scrape_result and hasattr(scrape_result, 'data'):
+                # Handle Firecrawl response object structure
+                data = scrape_result.data
+                markdown_content = getattr(data, 'markdown', '') or ''
+                html_content = getattr(data, 'html', '') or ''
+                metadata = getattr(data, 'metadata', {}) or {}
+                
+                # Create preview from metadata and content
+                title = metadata.get('title', 'Not specified') if isinstance(metadata, dict) else 'Not specified'
+                preview = {
+                    'company': 'Not specified',
+                    'role': title,
+                    'location': 'Not specified',
+                    'posted_days': 'Recently'
+                }
+                
+                # Try to extract better preview info from markdown content
+                enhanced_preview = self._extract_preview_from_markdown(markdown_content, url)
+                preview.update({k: v for k, v in enhanced_preview.items() if v != 'Not specified'})
+                
+                return {
+                    'success': True,
+                    'content': markdown_content or html_content,
+                    'html_content': html_content,
+                    'markdown_content': markdown_content,
+                    'metadata': metadata,
+                    'preview': preview,
+                    'url': url,
+                    'scraping_method': 'firecrawl'
+                }
+            else:
+                error_msg = getattr(scrape_result, 'error', 'Unknown Firecrawl error') if scrape_result else 'No response from Firecrawl'
+                return {
+                    'success': False,
+                    'error': f"Firecrawl failed: {error_msg}",
+                    'preview': {'company': 'Error', 'role': 'Firecrawl failed', 'location': '', 'posted_days': ''},
+                    'content': ''
+                }
+                
+        except Exception as e:
+            print(f"❌ Firecrawl error: {str(e)}")
+            # Fallback to other methods
+            if 'linkedin.com' in url:
+                return self._scrape_linkedin(url)
+            else:
+                return self._scrape_generic(url)
+    
+    def _extract_preview_from_markdown(self, markdown: str, url: str) -> dict:
+        """Extract preview info from Firecrawl markdown content"""
+        preview = {
+            'company': 'Not specified',
+            'role': 'Not specified',
+            'location': 'Not specified',
+            'posted_days': 'Recently'
+        }
+        
+        if not markdown:
+            return preview
+            
+        lines = markdown.split('\n')
+        
+        # Enhanced extraction for different job sites
+        for i, line in enumerate(lines[:15]):  # Check first 15 lines
+            line = line.strip()
+            if len(line) < 3:
+                continue
+                
+            # Extract from headers (usually job titles)
+            if line.startswith('#') and i < 5:
+                clean_title = line.lstrip('#').strip()
+                if not any(word in clean_title.lower() for word in ['about', 'company', 'description', 'overview']):
+                    preview['role'] = clean_title
+            
+            # LinkedIn specific patterns
+            if 'linkedin.com' in url:
+                # Company name after role
+                if ' at ' in line and preview['role'] != 'Not specified':
+                    parts = line.split(' at ')
+                    if len(parts) == 2:
+                        preview['company'] = parts[1].strip()
+                
+                # Location patterns
+                location_match = re.search(r'([^,]+,\s*[A-Z]{2}(?:\s*\d{5})?)', line)
+                if location_match:
+                    preview['location'] = location_match.group(1).strip()
+            
+            # General patterns for other sites
+            elif any(site in url for site in ['microsoft.com', 'google.com', 'amazon', 'meta.com']):
+                # Extract company from URL
+                if 'microsoft.com' in url:
+                    preview['company'] = 'Microsoft'
+                elif 'google.com' in url:
+                    preview['company'] = 'Google'
+                elif 'amazon' in url:
+                    preview['company'] = 'Amazon'
+                elif 'meta.com' in url:
+                    preview['company'] = 'Meta'
+                    
+                # Look for location in structured content
+                if re.search(r'\b(Remote|Hybrid|On-site)\b', line, re.IGNORECASE):
+                    preview['location'] = line.strip()
+        
+        return preview
+
     def _scrape_with_selenium(self, url: str) -> dict:
         """Selenium fallback for sites that block requests"""
         if not SELENIUM_AVAILABLE:

requirements.txt

@@ -22,3 +22,4 @@ google-auth-httplib2>=0.1.0
 authlib>=1.2.0
 selenium>=4.0.0
 webdriver-manager>=3.8.0
+firecrawl-py>=0.0.20