ok

app.py

@@ -4,6 +4,8 @@ import validators
 import os
 import tempfile
 import uuid
+from urllib.parse import urlparse
+from typing import Dict, Any
 
 # Import SEO modules
 from modules.technical_seo import TechnicalSEOModule
@@ -29,6 +31,77 @@ llm_recommendations = LLMRecommendations()
 # Store for generated reports (in production, use database)
 reports_store = {}
 
+def _transform_keywords_data(new_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Transform new keywords data structure to match report generator expectations"""
+    if not new_data or new_data.get('placeholder'):
+        return {
+            'placeholder': True,
+            'message': 'No keyword data available',
+            'total_keywords': 0,
+            'position_distribution': {'top_3': 0, 'top_10': 0, 'top_50': 0, 'beyond_50': 0},
+            'best_keywords': [],
+            'opportunity_keywords': [],
+            'data_source': 'Analysis failed'
+        }
+    
+    # Transform new structure to old structure
+    totals = new_data.get('totals', {})
+    distribution = new_data.get('distribution', {})
+    movement = new_data.get('movement', {})
+    best_keywords = new_data.get('best_keywords', [])
+    declining_keywords = new_data.get('declining_keywords', [])
+    opportunities = new_data.get('opportunities', [])
+    data_sources = new_data.get('data_sources', {})
+    
+    # Transform position distribution
+    pos_dist = {
+        'top_3': distribution.get('top3', 0),
+        'top_10': distribution.get('top10', 0),
+        'top_50': distribution.get('top50', 0),
+        'beyond_50': totals.get('keywords', 0) - distribution.get('top50', 0)
+    }
+    
+    # Transform best keywords to match expected format
+    transformed_best_keywords = []
+    for kw in best_keywords:
+        transformed_best_keywords.append({
+            'keyword': kw.get('keyword', ''),
+            'position': kw.get('rank', 0),
+            'clicks': 0,  # Not available in new API
+            'impressions': kw.get('volume', 0),
+            'url': kw.get('url', ''),
+            'estimated_traffic': kw.get('estimated_traffic', 0),
+            'trend': kw.get('trend', 'stable')
+        })
+    
+    # Transform opportunities to match expected format
+    transformed_opportunities = []
+    for opp in opportunities:
+        transformed_opportunities.append({
+            'keyword': opp.get('keyword', ''),
+            'position': 0,  # Not applicable for opportunities
+            'impressions': opp.get('volume', 0),
+            'ctr': 0,  # Not available
+            'competitor_rank': opp.get('competitor_rank', 0),
+            'priority_score': opp.get('priority_score', 0),
+            'competitor_domain': opp.get('competitor_domain', '')
+        })
+    
+    return {
+        'total_keywords': totals.get('keywords', 0),
+        'estimated_traffic': totals.get('estimated_traffic', 0),
+        'position_distribution': pos_dist,
+        'movement': movement,
+        'best_keywords': transformed_best_keywords,
+        'declining_keywords': declining_keywords,
+        'opportunity_keywords': transformed_opportunities,
+        'competitor_summary': new_data.get('competitor_summary', []),
+        'data_source': f"{data_sources.get('positions', 'Unknown')} + {data_sources.get('volume', 'Unknown')}",
+        'enrichment_rate': data_sources.get('enrichment_rate', 0),
+        'meta': new_data.get('meta', {}),
+        'placeholder': False
+    }
+
 @app.route('/')
 def index():
     return render_template('index.html')
@@ -49,12 +122,16 @@ def generate_report():
         # Generate unique report ID
         report_id = str(uuid.uuid4())
         
-        # Validate competitor URLs
+        # Validate competitor URLs and extract domains
+        competitor_domains = []
         competitor_list = []
         for comp in competitors:
             comp = comp.strip()
             if comp and validators.url(comp):
                 competitor_list.append(comp)
+                # Extract domain from competitor URL
+                domain = urlparse(comp).netloc.replace('www.', '')
+                competitor_domains.append(domain)
         
         # Technical SEO Analysis
         technical_data = technical_module.analyze(url)
@@ -62,53 +139,64 @@ def generate_report():
         # Content Audit
         content_data = content_module.analyze(url)
         
-        # Keywords Analysis
-        keywords_data = keywords_module.analyze(url).data
-        
-        # Backlinks Analysis - COMMENTED OUT TO SAVE API CREDITS
-        # print(f"DEBUG: Starting backlinks analysis for {url}")
-        # backlinks_result = backlinks_module.analyze(url)
-        # backlinks_data = backlinks_result.data
-        # print(f"DEBUG: Backlinks analysis result - Success: {backlinks_result.success}")
-        # print(f"DEBUG: Backlinks data keys: {list(backlinks_data.keys())}")
-        # if backlinks_data.get('total_backlinks'):
-        #     print(f"DEBUG: Total backlinks found: {backlinks_data.get('total_backlinks')}")
-        # if backlinks_data.get('placeholder'):
-        #     print(f"DEBUG: Using placeholder data: {backlinks_data.get('message')}")
-        
-        # Use placeholder backlinks data to save API credits
-        backlinks_data = {
-            'total_backlinks': 0,
-            'total_ref_domains': 0,
-            'domain_rating': 0,
-            'authority_scores': {'ahrefs_dr': 0, 'moz_da': 0, 'moz_pa': 0, 'majestic_tf': 0, 'majestic_cf': 0},
-            'referring_domains': [],
-            'anchor_distribution': [],
-            'monthly_changes': {'new_backlinks': 0, 'lost_backlinks': 0, 'net_change': 0},
-            'top_backlinks': [],
-            'quality_metrics': {'follow_ratio': 0, 'avg_authority': 0, 'quality_score': 0},
-            'edu_links': 0,
-            'gov_links': 0,
-            'estimated_organic_traffic': 0,
-            'organic_keywords': 0,
-            'data_sources': ['API disabled to save credits'],
-            'placeholder': True,
-            'message': 'Backlinks analysis temporarily disabled to conserve API credits.'
-        }
+        # Keywords Analysis - UPDATED: Pass competitor domains and handle errors
+        keywords_result = keywords_module.analyze(url, competitor_domains=competitor_domains)
+        if not keywords_result.success:
+            # Fallback to placeholder data if keywords analysis fails
+            keywords_data = {
+                'placeholder': True,
+                'message': f'Keywords analysis failed: {keywords_result.error}',
+                'total_keywords': 0,
+                'position_distribution': {'top_3': 0, 'top_10': 0, 'top_50': 0, 'beyond_50': 0},
+                'best_keywords': [],
+                'opportunity_keywords': [],
+                'data_source': 'Analysis failed'
+            }
+        else:
+            # Transform new data structure to match report generator expectations
+            keywords_data = _transform_keywords_data(keywords_result.data)
+        
+        # Backlinks Analysis - UNCOMMENTED: Enable backlinks analysis
+        print(f"DEBUG: Starting backlinks analysis for {url}")
+        backlinks_result = backlinks_module.analyze(url)
+        backlinks_data = backlinks_result.data
+        print(f"DEBUG: Backlinks analysis result - Success: {backlinks_result.success}")
+        print(f"DEBUG: Backlinks data keys: {list(backlinks_data.keys())}")
+        if backlinks_data.get('total_backlinks'):
+            print(f"DEBUG: Total backlinks found: {backlinks_data.get('total_backlinks')}")
+        if backlinks_data.get('placeholder'):
+            print(f"DEBUG: Using placeholder data: {backlinks_data.get('message')}")
         
         # Generate LLM Recommendations
         llm_rec_data = llm_recommendations.generate_recommendations(
             url, technical_data, content_data, keywords_data, backlinks_data
         )
         
-        # Competitor Analysis
+        # Competitor Analysis - UPDATED: Pass competitor domains
         competitor_data = []
         for comp_url in competitor_list:
             comp_technical = technical_module.analyze(comp_url)
             comp_content = content_module.analyze(comp_url, quick_scan=True)
-            comp_keywords = keywords_module.analyze(comp_url, quick_scan=True).data
-            # comp_backlinks = backlinks_module.analyze(comp_url, quick_scan=True).data  # SAVE API CREDITS
-            comp_backlinks = {'placeholder': True, 'message': 'Disabled to save credits'}
+            
+            # Keywords analysis for competitor (no competitors for competitor analysis)
+            comp_keywords_result = keywords_module.analyze(comp_url, competitor_domains=[], quick_scan=True)
+            if comp_keywords_result.success:
+                comp_keywords = _transform_keywords_data(comp_keywords_result.data)
+            else:
+                comp_keywords = {
+                    'placeholder': True, 
+                    'message': f'Keywords analysis failed: {comp_keywords_result.error}',
+                    'total_keywords': 0,
+                    'position_distribution': {'top_3': 0, 'top_10': 0, 'top_50': 0, 'beyond_50': 0},
+                    'best_keywords': [],
+                    'opportunity_keywords': [],
+                    'data_source': 'Analysis failed'
+                }
+            
+            # Backlinks analysis for competitor - UNCOMMENTED
+            comp_backlinks_result = backlinks_module.analyze(comp_url, quick_scan=True)
+            comp_backlinks = comp_backlinks_result.data
+            
             competitor_data.append({
                 'url': comp_url,
                 'technical': comp_technical,