{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4", "name": "OmniFile AI Processor v3.0 - Testing & Debugging" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 🧪 OmniFile AI Processor v3.0 — Testing & Debugging Notebook\n", "\n", "## 📋 الغرض من هذا الدفتر\n", "\n", "دفتر جوجل كولاب شامل لاختبار وتصحيح أخطاء جميع وحدات **OmniFile AI Processor v3.0**:\n", "\n", "| الوحدة | الوظيفة |\n", "|--------|--------|\n", "| 🔍 **OCR Engines** | اختبار 4 محركات (TrOCR, EasyOCR, Tesseract, PaddleOCR) |\n", "| 📊 **Fusion Strategies** | مقارنة 4 استراتيجيات دمج النتائج |\n", "| 📝 **NLP Pipeline** | اختبار التصحيح الإملائي والترجمة والتلخيص |\n", "| 📐 **Layout & Tables** | تحليل التخطيط واستخراج الجداول |\n", "| 🔐 **Security** | فحص PII والتشفير |\n", "| 📏 **Evaluation** | قياس CER/WER على بيانات الاختبار |\n", "| 🚀 **Performance** | قياس سرعة الاستدلال واستهلاك الذاكرة |\n", "| 🐛 **Debugging** | أدوات تشخيص الأخطاء الشاملة |" ] }, { "cell_type": "code", "metadata": { "id": "cell_0_info" }, "source": [ "# ============================================================\n", "# الخطوة 0: معلومات النظام والبيئة\n", "# ============================================================\n", "\n", "import sys\n", "import os\n", "import platform\n", "import subprocess\n", "from datetime import datetime\n", "\n", "print(\"=\" * 60)\n", "print(\"🧪 OmniFile AI Processor v3.0 — Testing & Debugging\")\n", "print(\"=\" * 60)\n", "print(f\"📅 التاريخ: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n", "print(f\"🐍 Python: {sys.version}\")\n", "print(f\"💻 النظام: {platform.system()} {platform.release()}\")\n", "print(f\"🖥️ المعالج: {platform.processor()}\")\n", "\n", "# فحص GPU\n", "try:\n", " import torch\n", " gpu_available = torch.cuda.is_available()\n", " if gpu_available:\n", " gpu_name = torch.cuda.get_device_name(0)\n", " gpu_mem = torch.cuda.get_device_properties(0).total_mem / 1e9\n", " print(f\"🎮 GPU: ✅ {gpu_name} ({gpu_mem:.1f} GB)\")\n", " else:\n", " print(\"🎮 GPU: ❌ غير متوفر\")\n", "except ImportError:\n", " print(\"🎮 GPU: ⚠️ PyTorch غير مثبت\")\n", " gpu_available = False\n", "\n", "# فحص CUDA\n", "try:\n", " cuda_version = subprocess.check_output([\"nvcc\", \"--version\"], stderr=subprocess.DEVNULL).decode()\n", " print(f\"⚡ CUDA: ✅\")\n", "except:\n", " print(f\"⚡ CUDA: ⚠️ غير متوفر\")\n", "\n", "# ذاكرة النظام\n", "try:\n", " import psutil\n", " ram = psutil.virtual_memory()\n", " print(f\"💾 RAM: {ram.total / 1e9:.1f} GB (متاح: {ram.available / 1e9:.1f} GB)\")\n", "except ImportError:\n", " print(\"💾 RAM: psutil غير مثبت\")\n", "\n", "print(\"=\" * 60)" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 1: استنساخ المشروع وتثبيت الاعتماديات" ] }, { "cell_type": "code", "metadata": { "id": "cell_1_clone" }, "source": [ "# ============================================================\n", "# الخطوة 1: استنساخ المشروع وتثبيت الاعتماديات\n", "# ============================================================\n", "\n", "import os\n", "\n", "# تغيير المجلد\n", "WORK_DIR = \"/content/OmniFile_Processor\"\n", "\n", "if not os.path.exists(WORK_DIR):\n", " print(\"📥 جاري استنساخ المشروع...\")\n", " !git clone /content/OmniFile_Processor\n", " os.chdir(WORK_DIR)\n", "else:\n", " print(\"✅ المشروع موجود بالفعل\")\n", " os.chdir(WORK_DIR)\n", "\n", "print(f\"📂 مجلد العمل: {os.getcwd()}\")\n", "print(f\"📁 المحتويات: {os.listdir('.')}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_1_install" }, "source": [ "# ============================================================\n", "# تثبيت الاعتماديات (مع تخطي المحركات غير المطلوبة)\n", "# ============================================================\n", "\n", "print(\"📦 جاري تثبيت الاعتماديات...\")\n", "\n", "!pip install -q torch torchvision --index-url https://download.pytorch.org/whl/cu121\n", "!pip install -q transformers peft datasets huggingface_hub sentencepiece\n", "!pip install -q easyocr pytesseract paddlepaddle paddleocr\n", "!pip install -q opencv-python-headless Pillow PyMuPDF pdf2image\n", "!pip install -q fastapi uvicorn numpy pandas scikit-learn\n", "!pip install -q pyspellchecker arabic-reshaper python-bidi langdetect\n", "!pip install -q rapidfuzz jiwer scikit-image\n", "!pip install -q openpyxl python-docx fpdf2 beautifulsoup4\n", "!pip install -q presidio-analyzer presidio-anonymizer detect-secrets\n", "!pip install -q cryptography onnxruntime psutil tqdm\n", "!pip install -q ar-corrector language-tool-python\n", "!pip install -q pydantic pydantic-settings requests python-multipart\n", "\n", "# تثبيت Tesseract (للنظام)\n", "!apt-get install -q -y tesseract-ocr tesseract-ocr-arab tesseract-ocr-eng\n", "\n", "print(\"\\n✅ تم تثبيت جميع الاعتماديات!\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 2: فحص توفر المحركات والمكتبات" ] }, { "cell_type": "code", "metadata": { "id": "cell_2_check" }, "source": [ "# ============================================================\n", "# الخطوة 2: فحص توفر جميع المحركات والمكتبات\n", "# ============================================================\n", "\n", "import importlib\n", "\n", "def check_lib(name, package=None):\n", " \"\"\"فحص توفر مكتبة\"\"\"\n", " try:\n", " mod = importlib.import_module(name)\n", " ver = getattr(mod, '__version__', 'N/A')\n", " return True, ver\n", " except ImportError:\n", " return False, None\n", "\n", "libraries = {\n", " 'PyTorch': ('torch', None),\n", " 'Transformers': ('transformers', None),\n", " 'EasyOCR': ('easyocr', None),\n", " 'PaddleOCR': ('paddleocr', None),\n", " 'OpenCV': ('cv2', 'opencv-python-headless'),\n", " 'PIL': ('PIL', 'Pillow'),\n", " 'PyMuPDF': ('fitz', 'PyMuPDF'),\n", " 'pdf2image': ('pdf2image', None),\n", " 'ONNX Runtime': ('onnxruntime', None),\n", " 'arabic_reshaper': ('arabic_reshaper', None),\n", " 'bidi': ('bidi', 'python-bidi'),\n", " 'langdetect': ('langdetect', None),\n", " 'Presidio': ('presidio_analyzer', None),\n", " 'cryptography': ('cryptography', None),\n", " 'psutil': ('psutil', None),\n", " 'Pydantic': ('pydantic', None),\n", " 'FastAPI': ('fastapi', None),\n", " 'scikit-learn': ('sklearn', 'scikit-learn'),\n", " 'jiwer': ('jiwer', None),\n", " 'rapidfuzz': ('rapidfuzz', None),\n", "}\n", "\n", "print(\"📋 فحص المكتبات والاعتماديات\")\n", "print(\"=\" * 50)\n", "\n", "available = 0\n", "for name, (mod, pkg) in libraries.items():\n", " ok, ver = check_lib(mod)\n", " status = f\"✅ v{ver}\" if ok else \"❌ غير مثبت\"\n", " print(f\" {name:20s} {status}\")\n", " if ok:\n", " available += 1\n", "\n", "# فحص Tesseract\n", "import shutil\n", "tesseract_path = shutil.which('tesseract')\n", "tesseract_status = f\"✅ {tesseract_path}\" if tesseract_path else \"❌ غير مثبت\"\n", "print(f\" {'Tesseract':20s} {tesseract_status}\")\n", "if tesseract_path:\n", " available += 1\n", "\n", "print(f\"\\n📊 الإجمالي: {available}/{len(libraries)+1} مكتبة متوفرة\")\n", "print(\"=\" * 50)" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 3: إعداد الإعدادات وتهيئة النظام" ] }, { "cell_type": "code", "metadata": { "id": "cell_3_config" }, "source": [ "# ============================================================\n", "# الخطوة 3: إعداد الإعدادات\n", "# ============================================================\n", "\n", "import sys\n", "sys.path.insert(0, WORK_DIR)\n", "\n", "from config import OmniFileConfig\n", "\n", "# إعداد البيئة\n", "config = OmniFileConfig.from_colab_drive() if os.path.exists('/content/drive/MyDrive') else OmniFileConfig()\n", "config.environment = \"colab\"\n", "config.ensure_dirs()\n", "config.setup_environment()\n", "\n", "print(\"⚙️ الإعدادات\")\n", "print(f\" البيئة: {config.environment}\")\n", "print(f\" OCR: TrOCR={config.enable_trocr}, EasyOCR={config.enable_easyocr}, Tesseract={config.enable_tesseract}, PaddleOCR={config.enable_paddleocr}\")\n", "print(f\" ONNX: {config.use_onnx}\")\n", "print(f\" Quantization: {config.use_quantization}\")\n", "print(f\" DPI: {config.dpi}\")\n", "print(f\" GPU: {config.use_gpu if hasattr(config, 'use_gpu') else 'auto'}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 4: اختبار محركات OCR (كل محرك على حدة)" ] }, { "cell_type": "code", "metadata": { "id": "cell_4_sample" }, "source": [ "# ============================================================\n", "# الخطوة 4a: إنشاء صور اختبار (عربي + إنجليزي + مختلط)\n", "# ============================================================\n", "\n", "import numpy as np\n", "from PIL import Image, ImageDraw, ImageFont\n", "\n", "def create_test_image(text, filename, width=800, height=200):\n", " \"\"\"إنشاء صورة اختبار بنص معين\"\"\"\n", " img = Image.new('RGB', (width, height), 'white')\n", " draw = ImageDraw.Draw(img)\n", " try:\n", " font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 28)\n", " except:\n", " font = ImageFont.load_default()\n", " draw.text((30, 80), text, fill='black', font=font)\n", " img.save(filename)\n", " return filename\n", "\n", "# إنشاء صور اختبار\n", "os.makedirs('test_samples', exist_ok=True)\n", "\n", "samples = {\n", " 'arabic': 'مرحبا بالعالم - نص عربي تجريبي',\n", " 'english': 'Hello World - This is a test text for OCR',\n", " 'mixed': 'Arabic مرحبا and English mixed text 123',\n", " 'numbers': '0123456789 - ١٢٣٤٥٦٧٨٩٠',\n", "}\n", "\n", "for name, text in samples.items():\n", " path = create_test_image(text, f'test_samples/{name}.png')\n", " print(f\"✅ تم إنشاء: test_samples/{name}.png\")\n", "\n", "# عرض الصور\n", "from IPython.display import display\n", "for name in samples:\n", " img = Image.open(f'test_samples/{name}.png')\n", " display(img)" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_4_tesseract" }, "source": [ "# ============================================================\n", "# الخطوة 4b: اختبار محرك Tesseract\n", "# ============================================================\n", "\n", "import pytesseract\n", "from PIL import Image\n", "import time\n", "\n", "print(\"🔍 اختبار محرك Tesseract\")\n", "print(\"=\" * 50)\n", "\n", "tesseract_results = {}\n", "\n", "for name, expected in samples.items():\n", " img = Image.open(f'test_samples/{name}.png')\n", " \n", " start = time.time()\n", " text = pytesseract.image_to_string(img, lang='ara+eng')\n", " elapsed = time.time() - start\n", " \n", " tesseract_results[name] = {\n", " 'expected': expected,\n", " 'result': text.strip(),\n", " 'time': elapsed\n", " }\n", " \n", " match = \"✅\" if text.strip() == expected else \"⚠️\"\n", " print(f\"\\n{match} اختبار {name} ({elapsed:.3f}s):\")\n", " print(f\" المتوقع: {expected}\")\n", " print(f\" الناتج: {text.strip()}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_4_easyocr" }, "source": [ "# ============================================================\n", "# الخطوة 4c: اختبار محرك EasyOCR\n", "# ============================================================\n", "\n", "import easyocr\n", "import time\n", "import numpy as np\n", "\n", "print(\"🔍 اختبار محرك EasyOCR\")\n", "print(\"=\" * 50)\n", "\n", "reader = easyocr.Reader(['ar', 'en'], gpu=gpu_available, verbose=False)\n", "\n", "easyocr_results = {}\n", "\n", "for name, expected in samples.items():\n", " img = np.array(Image.open(f'test_samples/{name}.png'))\n", " \n", " start = time.time()\n", " result = reader.readtext(img)\n", " elapsed = time.time() - start\n", " \n", " text = ' '.join([item[1] for item in result])\n", " easyocr_results[name] = {\n", " 'expected': expected,\n", " 'result': text.strip(),\n", " 'time': elapsed,\n", " 'details': result\n", " }\n", " \n", " match = \"✅\" if text.strip() == expected else \"⚠️\"\n", " print(f\"\\n{match} اختبار {name} ({elapsed:.3f}s):\")\n", " print(f\" المتوقع: {expected}\")\n", " print(f\" الناتج: {text.strip()}\")\n", " if result:\n", " print(f\" التفاصيل: {len(result)} منطقة\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_4_trocr" }, "source": [ "# ============================================================\n", "# الخطوة 4d: اختبار محرك TrOCR\n", "# ============================================================\n", "\n", "from transformers import TrOCRProcessor, VisionEncoderDecoderModel\n", "from PIL import Image\n", "import time\n", "\n", "print(\"🔍 اختبار محرك TrOCR\")\n", "print(\"=\" * 50)\n", "\n", "# تحميل النموذج\n", "processor = TrOCRProcessor.from_pretrained('microsoft/trocr-base-handwritten')\n", "model = VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-base-handwritten')\n", "if gpu_available:\n", " model = model.cuda()\n", "\n", "trocr_results = {}\n", "\n", "for name, expected in samples.items():\n", " img = Image.open(f'test_samples/{name}.png').convert('RGB')\n", " \n", " start = time.time()\n", " pixel_values = processor(img, return_tensors='pt').pixel_values\n", " if gpu_available:\n", " pixel_values = pixel_values.cuda()\n", " generated_ids = model.generate(pixel_values)\n", " text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]\n", " elapsed = time.time() - start\n", " \n", " trocr_results[name] = {\n", " 'expected': expected,\n", " 'result': text.strip(),\n", " 'time': elapsed\n", " }\n", " \n", " match = \"✅\" if text.strip() == expected else \"⚠️\"\n", " print(f\"\\n{match} اختبار {name} ({elapsed:.3f}s):\")\n", " print(f\" المتوقع: {expected}\")\n", " print(f\" الناتج: {text.strip()}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_4_paddleocr" }, "source": [ "# ============================================================\n", "# الخطوة 4e: اختبار محرك PaddleOCR\n", "# ============================================================\n", "\n", "from paddleocr import PaddleOCR\n", "import time\n", "import numpy as np\n", "\n", "print(\"🔍 اختبار محرك PaddleOCR\")\n", "print(\"=\" * 50)\n", "\n", "paddle_ocr = PaddleOCR(use_angle_cls=True, lang='ar', show_log=False)\n", "\n", "paddleocr_results = {}\n", "\n", "for name, expected in samples.items():\n", " img = np.array(Image.open(f'test_samples/{name}.png'))\n", " \n", " start = time.time()\n", " result = paddle_ocr.ocr(img, cls=True)\n", " elapsed = time.time() - start\n", " \n", " text_parts = []\n", " if result and result[0]:\n", " for line in result[0]:\n", " text_parts.append(line[1][0])\n", " text = ' '.join(text_parts)\n", " \n", " paddleocr_results[name] = {\n", " 'expected': expected,\n", " 'result': text.strip(),\n", " 'time': elapsed\n", " }\n", " \n", " match = \"✅\" if text.strip() == expected else \"⚠️\"\n", " print(f\"\\n{match} اختبار {name} ({elapsed:.3f}s):\")\n", " print(f\" المتوقع: {expected}\")\n", " print(f\" الناتج: {text.strip()}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 5: مقارنة النتائج واختبار استراتيجيات الدمج" ] }, { "cell_type": "code", "metadata": { "id": "cell_5_compare" }, "source": [ "# ============================================================\n", "# الخطوة 5: مقارنة شاملة لجميع المحركات\n", "# ============================================================\n", "\n", "import pandas as pd\n", "\n", "print(\"📊 مقارنة شاملة لنتائج المحركات\")\n", "print(\"=\" * 70)\n", "\n", "# بناء جدول المقارنة\n", "comparison = []\n", "for name, expected in samples.items():\n", " row = {'Test': name, 'Expected': expected}\n", " for engine, results in [\n", " ('Tesseract', tesseract_results),\n", " ('EasyOCR', easyocr_results),\n", " ('TrOCR', trocr_results),\n", " ('PaddleOCR', paddleocr_results)\n", " ]:\n", " if name in results:\n", " row[f'{engine} Result'] = results[name]['result'][:40]\n", " row[f'{engine} Time'] = f\"{results[name]['time']:.3f}s\"\n", " comparison.append(row)\n", "\n", "df = pd.DataFrame(comparison)\n", "pd.set_option('display.max_colwidth', 50)\n", "pd.set_option('display.width', 200)\n", "display(df)" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_5_fusion" }, "source": [ "# ============================================================\n", "# الخطوة 5b: اختبار استراتيجيات الدمج\n", "# ============================================================\n", "\n", "sys.path.insert(0, WORK_DIR)\n", "from modules.vision.result_fusion import ResultFusion, FusionStrategy\n", "\n", "print(\"🔀 اختبار استراتيجيات الدمج\")\n", "print(\"=\" * 50)\n", "\n", "fusion = ResultFusion()\n", "\n", "for name, expected in samples.items():\n", " print(f\"\\n📝 اختبار: {name}\")\n", " print(f\" المتوقع: {expected}\")\n", " print(f\" {'-' * 50}\")\n", " \n", " # جمع نتائج جميع المحركات\n", " engine_results = []\n", " for engine, results, confidence in [\n", " ('Tesseract', tesseract_results, 0.7),\n", " ('EasyOCR', easyocr_results, 0.85),\n", " ('TrOCR', trocr_results, 0.9),\n", " ('PaddleOCR', paddleocr_results, 0.8)\n", " ]:\n", " if name in results:\n", " engine_results.append({\n", " 'engine': engine,\n", " 'text': results[name]['result'],\n", " 'confidence': confidence\n", " })\n", " \n", " # اختبار كل استراتيجية\n", " for strategy in FusionStrategy:\n", " try:\n", " fused = fusion.fuse(engine_results, strategy=strategy)\n", " print(f\" {strategy.value:25s}: {fused[:60]}\")\n", " except Exception as e:\n", " print(f\" {strategy.value:25s}: ❌ خطأ: {e}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 6: اختبار وحدة OCREngine الموحدة" ] }, { "cell_type": "code", "metadata": { "id": "cell_6_engine" }, "source": [ "# ============================================================\n", "# الخطوة 6: اختبار محرك OCR الموحد\n", "# ============================================================\n", "\n", "from modules.vision.ocr_engine import OCREngine\n", "import time\n", "\n", "print(\"🧠 اختبار محرك OCR الموحد\")\n", "print(\"=\" * 50)\n", "\n", "engine = OCREngine(\n", " enable_trocr=True,\n", " enable_easyocr=True,\n", " enable_tesseract=True,\n", " enable_paddleocr=True,\n", " use_gpu=gpu_available\n", ")\n", "\n", "print(f\"\\n📊 المحركات المتاحة: {engine._get_active_engines()}\")\n", "\n", "for name, expected in samples.items():\n", " img = Image.open(f'test_samples/{name}.png')\n", " \n", " start = time.time()\n", " result = engine.recognize(img, engine='all')\n", " elapsed = time.time() - start\n", " \n", " text = ' '.join([token.text for token in result]) if result else '(فارغ)'\n", " print(f\"\\n{'='*50}\")\n", " print(f\"📝 {name} ({elapsed:.3f}s):\")\n", " print(f\" المتوقع: {expected}\")\n", " print(f\" الناتج: {text}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 7: اختبار تحليل التخطيط واستخراج الجداول" ] }, { "cell_type": "code", "metadata": { "id": "cell_7_layout" }, "source": [ "# ============================================================\n", "# الخطوة 7a: اختبار تحليل التخطيط\n", "# ============================================================\n", "\n", "from modules.vision.layout_analyzer import LayoutAnalyzer\n", "import numpy as np\n", "import cv2\n", "\n", "print(\"📐 اختبار تحليل التخطيط\")\n", "print(\"=\" * 50)\n", "\n", "analyzer = LayoutAnalyzer()\n", "\n", "# إنشاء صورة مستند تجريبية مع نص وجدول\n", "doc_img = Image.new('RGB', (800, 1000), 'white')\n", "draw = ImageDraw.Draw(doc_img)\n", "\n", "try:\n", " font = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 20)\n", " font_small = ImageFont.truetype('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf', 14)\n", "except:\n", " font = ImageFont.load_default()\n", " font_small = font\n", "\n", "# رسم عنوان\n", "draw.text((50, 30), 'Document Title - عنوان المستند', fill='black', font=font)\n", "\n", "# رسم فقرة\n", "draw.text((50, 80), 'This is a paragraph of text for testing layout analysis.', fill='black', font=font_small)\n", "draw.text((50, 100), 'هذا نص عربي لاختبار تحليل التخطيط.', fill='black', font=font_small)\n", "\n", "# رسم جدول بسيط\n", "table_x, table_y = 50, 150\n", "for i in range(4):\n", " y = table_y + i * 40\n", " draw.line([(table_x, y), (table_x + 700, y)], fill='black', width=2)\n", "for i in range(3):\n", " x = table_x + i * 233\n", " draw.line([(x, table_y), (x, table_y + 160)], fill='black', width=2)\n", "\n", "# كتابة نص في الخلايا\n", "headers = ['Column 1', 'العمود الثاني', 'Column 3']\n", "for i, h in enumerate(headers):\n", " draw.text((table_x + i * 233 + 10, table_y + 10), h, fill='black', font=font_small)\n", "\n", "doc_img.save('test_samples/layout_test.png')\n", "\n", "# تحليل التخطيط\n", "img_array = np.array(doc_img)\n", "blocks = analyzer.analyze(img_array)\n", "\n", "print(f\"\\n📊 تم اكتشاف {len(blocks)} كتل:\")\n", "for block in blocks:\n", " print(f\" - {block.block_type.value:15s} | الثقة: {block.confidence:.2f} | الموقع: ({block.bbox.x}, {block.bbox.y})\")\n", "\n", "display(doc_img)" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_7_table" }, "source": [ "# ============================================================\n", "# الخطوة 7b: اختبار استخراج الجداول\n", "# ============================================================\n", "\n", "from modules.vision.table_extractor import TableExtractor\n", "from modules.core.structure import BBox\n", "\n", "print(\"📋 اختبار استخراج الجداول\")\n", "print(\"=\" * 50)\n", "\n", "# تحديد منطقة الجدول يدوياً\n", "table_bbox = BBox(x=50, y=150, width=700, height=160)\n", "\n", "extractor = TableExtractor(ocr_engine=engine)\n", "table_data = extractor.extract_table(img_array, table_bbox)\n", "\n", "print(f\"\\n📊 بيانات الجدول المستخرجة:\")\n", "for i, row in enumerate(table_data):\n", " print(f\" صف {i}: {row}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 8: اختبار وحدة NLP (التصحيح الإملائي والترجمة)" ] }, { "cell_type": "code", "metadata": { "id": "cell_8_nlp" }, "source": [ "# ============================================================\n", "# الخطوة 8: اختبار وحدة NLP\n", "# ============================================================\n", "\n", "print(\"📝 اختبار وحدة NLP\")\n", "print(\"=\" * 50)\n", "\n", "# 8a: التصحيح الإملائي\n", "print(\"\\n🔍 التصحيح الإملائي:\")\n", "try:\n", " from modules.nlp.spell_corrector import SpellCorrector\n", " corrector = SpellCorrector()\n", " \n", " test_texts = [\n", " ('en', 'Ths is a tst txt for spll chcking'),\n", " ('ar', 'هذا نصر تجريبي لاختبار التصحيح'),\n", " ]\n", " \n", " for lang, text in test_texts:\n", " corrected = corrector.correct(text, language=lang)\n", " print(f\" {lang}: '{text}' → '{corrected}'\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", "\n", "# 8b: كشف اللغة\n", "print(\"\\n🌍 كشف اللغة:\")\n", "try:\n", " from modules.nlp.language_detector import LanguageDetector\n", " detector = LanguageDetector()\n", " \n", " lang_samples = [\n", " 'Hello World',\n", " 'مرحبا بالعالم',\n", " 'Hallo Welt',\n", " 'Mixed text مع عربي',\n", " ]\n", " \n", " for text in lang_samples:\n", " lang = detector.detect(text)\n", " print(f\" '{text[:30]}' → {lang}\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", "\n", "# 8c: الترجمة\n", "print(\"\\n🌐 الترجمة:\")\n", "try:\n", " from modules.nlp.translator import TextTranslator\n", " translator = TextTranslator()\n", " \n", " translated = translator.translate('Hello World', source='en', target='ar')\n", " print(f\" EN→AR: 'Hello World' → '{translated}'\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", "\n", "# 8d: إعادة بناء النص العربي\n", "print(\"\\n🔄 إعادة بناء النص العربي:\")\n", "try:\n", " from modules.vision.text_reconstructor import TextReconstructor\n", " reconstructor = TextReconstructor()\n", " \n", " words = [\n", " {'text': 'مستند', 'x': 0, 'y': 0, 'w': 60, 'h': 25},\n", " {'text': 'تجريبي', 'x': 70, 'y': 0, 'w': 60, 'h': 25},\n", " ]\n", " text = reconstructor.reconstruct(words, direction='rtl')\n", " print(f\" RTL: {words} → '{text}'\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 9: اختبار وحدة التقييم (CER/WER)" ] }, { "cell_type": "code", "metadata": { "id": "cell_9_eval" }, "source": [ "# ============================================================\n", "# الخطوة 9: اختبار وحدة التقييم\n", "# ============================================================\n", "\n", "from modules.evaluation.metrics import evaluate, calculate_cer, calculate_wer\n", "\n", "print(\"📏 اختبار وحدة التقييم\")\n", "print(\"=\" * 50)\n", "\n", "eval_tests = [\n", " ('Perfect match', 'مرحبا بالعالم', 'مرحبا بالعالم'),\n", " ('Minor errors', 'التعرف على النصوص', 'التعرف على المصوص'),\n", " ('Major errors', 'معالجة الصور الرقمية', 'ملالة الصور الرقمية'),\n", " ('English text', 'OCR Accuracy Test', 'OCR Accurcy Test'),\n", " ('Mixed', 'نظام OCR متقدم', 'نظا OCR متقدم'),\n", "]\n", "\n", "eval_results = []\n", "\n", "for name, ref, hyp in eval_tests:\n", " result = evaluate(ref, hyp)\n", " eval_results.append({\n", " 'Test': name,\n", " 'CER': f\"{result.cer:.2%}\",\n", " 'WER': f\"{result.wer:.2%}\",\n", " 'Accuracy': f\"{result.accuracy_percent:.1f}%\",\n", " 'Grade': result.quality_grade\n", " })\n", " print(f\"\\n📝 {name}:\")\n", " print(f\" المرجع: {ref}\")\n", " print(f\" الفرضية: {hyp}\")\n", " print(f\" CER: {result.cer:.2%} | WER: {result.wer:.2%} | {result.quality_grade}\")\n", "\n", "print(f\"\\n{'='*50}\")\n", "print(\"\\n📊 جدول النتائج:\")\n", "eval_df = pd.DataFrame(eval_results)\n", "display(eval_df)" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 10: اختبار وحدة الأمان" ] }, { "cell_type": "code", "metadata": { "id": "cell_10_security" }, "source": [ "# ============================================================\n", "# الخطوة 10: اختبار وحدة الأمان\n", "# ============================================================\n", "\n", "print(\"🔐 اختبار وحدة الأمان\")\n", "print(\"=\" * 50)\n", "\n", "# 10a: فحص PII\n", "print(\"\\n🔍 فحص البيانات الشخصية الحساسة (PII):\")\n", "try:\n", " from modules.security.sensitive_data_scanner import SensitiveDataScanner\n", " scanner = SensitiveDataScanner()\n", " \n", " pii_tests = [\n", " 'اسمي أحمد محمد وهاتفي 0912345678',\n", " 'My email is test@example.com',\n", " 'Credit card: 4532-1234-5678-9010',\n", " ]\n", " \n", " for text in pii_tests:\n", " result = scanner.scan_text(text)\n", " print(f\" '{text}' → {len(result)} كيانات حساسة\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", "\n", "# 10b: التشفير\n", "print(\"\\n🔒 اختبار التشفير:\")\n", "try:\n", " from modules.security.encryption import FileEncryptor\n", " encryptor = FileEncryptor()\n", " \n", " # إنشاء ملف اختبار\n", " with open('test_samples/encrypt_test.txt', 'w') as f:\n", " f.write('هذا نص سري للتجربة')\n", " \n", " encryptor.encrypt_file('test_samples/encrypt_test.txt', 'test_password')\n", " print(\" ✅ تم التشفير\")\n", " \n", " encryptor.decrypt_file('test_samples/encrypt_test.txt.enc', 'test_password')\n", " print(\" ✅ تم فك التشفير\")\n", " \n", " with open('test_samples/encrypt_test.txt.dec', 'r') as f:\n", " decrypted = f.read()\n", " print(f\" النص الأصلي: هذا نص سري للتجربة\")\n", " print(f\" بعد فك التشفير: {decrypted}\")\n", " match = decrypted == 'هذا نص سري للتجربة'\n", " print(f\" {'✅ متطابق' if match else '❌ غير متطابق'}\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 11: اختبار معالجة PDF" ] }, { "cell_type": "code", "metadata": { "id": "cell_11_pdf" }, "source": [ "# ============================================================\n", "# الخطوة 11: اختبار معالجة PDF\n", "# ============================================================\n", "\n", "print(\"📄 اختبار معالجة PDF\")\n", "print(\"=\" * 50)\n", "\n", "import fitz # PyMuPDF\n", "\n", "# إنشاء ملف PDF اختبار\n", "pdf_doc = fitz.open()\n", "page = pdf_doc.new_page(width=595, height=842) # A4\n", "\n", "# إضافة نص عربي وإنجليزي\n", "page.insert_text((50, 100), 'Document Title - عنوان المستند', fontsize=18)\n", "page.insert_text((50, 150), 'This is a test paragraph for PDF processing.', fontsize=12)\n", "page.insert_text((50, 180), 'هذا نص عربي لاختبار معالجة ملفات PDF.', fontsize=12)\n", "\n", "pdf_path = 'test_samples/test_doc.pdf'\n", "pdf_doc.save(pdf_path)\n", "pdf_doc.close()\n", "\n", "print(f\"\\n✅ تم إنشاء: {pdf_path}\")\n", "\n", "# معالجة PDF باستخدام OCREngine\n", "try:\n", " start = time.time()\n", " results = engine.recognize_pdf(pdf_path)\n", " elapsed = time.time() - start\n", " \n", " print(f\"\\n📊 نتائج OCR لل PDF ({elapsed:.3f}s):\")\n", " for i, page_results in enumerate(results):\n", " text = ' '.join([t.text for t in page_results])\n", " print(f\" صفحة {i+1}: {text[:100]}...\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", " import traceback\n", " traceback.print_exc()" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 12: اختبار الأداء والاستهلاك" ] }, { "cell_type": "code", "metadata": { "id": "cell_12_perf" }, "source": [ "# ============================================================\n", "# الخطوة 12: قياس الأداء واستهلاك الموارد\n", "# ============================================================\n", "\n", "import gc\n", "import time\n", "import psutil\n", "\n", "def measure_performance(engine, image_path, engine_name, iterations=3):\n", " \"\"\"قياس أداء محرك OCR\"\"\"\n", " img = Image.open(image_path)\n", " \n", " times = []\n", " memories = []\n", " \n", " for _ in range(iterations):\n", " gc.collect()\n", " \n", " mem_before = psutil.Process().memory_info().rss / 1e6\n", " start = time.time()\n", " \n", " result = engine.recognize(img, engine=engine_name)\n", " \n", " elapsed = time.time() - start\n", " mem_after = psutil.Process().memory_info().rss / 1e6\n", " \n", " times.append(elapsed)\n", " memories.append(mem_after - mem_before)\n", " \n", " return {\n", " 'avg_time': sum(times) / len(times),\n", " 'min_time': min(times),\n", " 'max_time': max(times),\n", " 'avg_mem': sum(memories) / len(memories),\n", " }\n", "\n", "print(\"⚡ اختبار الأداء\")\n", "print(\"=\" * 60)\n", "\n", "perf_engines = ['tesseract', 'easyocr'] # محركات سريعة فقط\n", "\n", "for eng in perf_engines:\n", " try:\n", " perf = measure_performance(engine, 'test_samples/arabic.png', eng)\n", " print(f\"\\n🔍 {eng.upper()}:\")\n", " print(f\" ⏱️ الوقت: {perf['avg_time']:.3f}s (min: {perf['min_time']:.3f}s, max: {perf['max_time']:.3f}s)\")\n", " print(f\" 💾 الذاكرة: +{perf['avg_mem']:.1f} MB\")\n", " except Exception as e:\n", " print(f\"\\n🔍 {eng.upper()}: ❌ {e}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 13: أدوات التشخيص وإصلاح الأخطاء" ] }, { "cell_type": "code", "metadata": { "id": "cell_13_debug" }, "source": [ "# ============================================================\n", "# الخطوة 13: أدوات التشخيص الشاملة\n", "# ============================================================\n", "\n", "import traceback\n", "import warnings\n", "\n", "print(\"🐛 أدوات التشخيص\")\n", "print(\"=\" * 60)\n", "\n", "# 13a: فحص إعدادات Tesseract\n", "print(\"\\n🔍 فحص Tesseract:\")\n", "try:\n", " import pytesseract\n", " print(f\" الإصدار: {pytesseract.get_tesseract_version()}\")\n", " langs = pytesseract.get_languages()\n", " print(f\" اللغات: {langs}\")\n", " print(f\" المسار: {pytesseract.pytesseract.tesseract_cmd}\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", "\n", "# 13b: فحص إعدادات GPU\n", "print(\"\\n🎮 فحص GPU:\")\n", "try:\n", " import torch\n", " print(f\" CUDA متاح: {torch.cuda.is_available()}\")\n", " if torch.cuda.is_available():\n", " print(f\" عدد الأجهزة: {torch.cuda.device_count()}\")\n", " print(f\" اسم الجهاز: {torch.cuda.get_device_name(0)}\")\n", " print(f\" ذاكرة GPU: {torch.cuda.get_device_properties(0).total_mem / 1e9:.1f} GB\")\n", " print(f\" CUDA Version: {torch.version.cuda}\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", "\n", "# 13c: فحص الذاكرة\n", "print(\"\\n💾 فحص الذاكرة:\")\n", "try:\n", " import psutil\n", " proc = psutil.Process()\n", " mem = proc.memory_info()\n", " print(f\" RSS: {mem.rss / 1e6:.1f} MB\")\n", " print(f\" VMS: {mem.vms / 1e6:.1f} MB\")\n", " if torch.cuda.is_available():\n", " print(f\" GPU Memory: {torch.cuda.memory_allocated() / 1e6:.1f} MB\")\n", " print(f\" GPU Cached: {torch.cuda.memory_reserved() / 1e6:.1f} MB\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")\n", "\n", "# 13d: فحص بيئة HuggingFace\n", "print(\"\\n🤗 فحص HuggingFace:\")\n", "try:\n", " from huggingface_hub import HfFolder\n", " token = HfFolder.get_token()\n", " print(f\" Token: {'✅ موجود' if token else '❌ غير موجود'}\")\n", " import os\n", " cache_dir = os.path.expanduser('~/.cache/huggingface')\n", " if os.path.exists(cache_dir):\n", " total_size = sum(\n", " os.path.getsize(os.path.join(dp, f))\n", " for dp, dn, filenames in os.walk(cache_dir)\n", " for f in filenames\n", " )\n", " print(f\" حجم الكاش: {total_size / 1e9:.2f} GB\")\n", "except Exception as e:\n", " print(f\" ❌ خطأ: {e}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "code", "metadata": { "id": "cell_13_full_test" }, "source": [ "# ============================================================\n", "# الخطوة 13e: اختبار شامل للنظام (Full Pipeline Test)\n", "# ============================================================\n", "\n", "print(\"🧪 اختبار شامل للنظام (End-to-End)\")\n", "print(\"=\" * 60)\n", "\n", "def full_pipeline_test(image_path, ground_truth=\"\"):\n", " \"\"\"اختبار كامل لخط المعالجة\"\"\"\n", " results = {}\n", " \n", " # 1. تحليل التخطيط\n", " try:\n", " img = np.array(Image.open(image_path))\n", " blocks = analyzer.analyze(img)\n", " results['layout'] = {'blocks': len(blocks), 'status': '✅'}\n", " except Exception as e:\n", " results['layout'] = {'error': str(e), 'status': '❌'}\n", " \n", " # 2. OCR\n", " try:\n", " img = Image.open(image_path)\n", " tokens = engine.recognize(img, engine='all')\n", " text = ' '.join([t.text for t in tokens])\n", " results['ocr'] = {'text': text[:50], 'tokens': len(tokens), 'status': '✅'}\n", " except Exception as e:\n", " results['ocr'] = {'error': str(e), 'status': '❌'}\n", " \n", " # 3. التصحيح الإملائي\n", " try:\n", " if results['ocr']['status'] == '✅':\n", " corrected = corrector.correct(text)\n", " results['spellcheck'] = {'corrected': corrected[:50], 'status': '✅'}\n", " except Exception as e:\n", " results['spellcheck'] = {'error': str(e), 'status': '❌'}\n", " \n", " # 4. التقييم\n", " if ground_truth:\n", " try:\n", " eval_result = evaluate(ground_truth, text)\n", " results['evaluation'] = {\n", " 'cer': f\"{eval_result.cer:.2%}\",\n", " 'wer': f\"{eval_result.wer:.2%}\",\n", " 'grade': eval_result.quality_grade,\n", " 'status': '✅'\n", " }\n", " except Exception as e:\n", " results['evaluation'] = {'error': str(e), 'status': '❌'}\n", " \n", " return results\n", "\n", "# تشغيل الاختبار الشامل\n", "for name in samples:\n", " print(f\"\\n{'='*60}\")\n", " print(f\"📝 اختبار شامل: {name}\")\n", " print(f\"{'='*60}\")\n", " \n", " results = full_pipeline_test(f'test_samples/{name}.png')\n", " \n", " for step, data in results.items():\n", " print(f\" {step:15s} {data['status']}\")\n", " if 'error' in data:\n", " print(f\" خطأ: {data['error']}\")\n", " elif 'text' in data:\n", " print(f\" نص: {data['text']}\")\n", " elif 'cer' in data:\n", " print(f\" CER: {data['cer']} | WER: {data['wer']} | {data['grade']}\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 14: اختبار ONNX Runtime والتسريع" ] }, { "cell_type": "code", "metadata": { "id": "cell_14_onnx" }, "source": [ "# ============================================================\n", "# الخطوة 14: اختبار ONNX Runtime\n", "# ============================================================\n", "\n", "print(\"⚡ اختبار ONNX Runtime\")\n", "print(\"=\" * 50)\n", "\n", "try:\n", " import onnxruntime as ort\n", " print(f\" ONNX Runtime: ✅ v{ort.__version__}\")\n", " print(f\" Providers: {ort.get_available_providers()}\")\n", " \n", " # تحميل TrOCR كـ ONNX\n", " print(\"\\n🔄 تحويل TrOCR إلى ONNX...\")\n", " try:\n", " onnx_path = engine.load_trocr_onnx()\n", " print(f\" ✅ تم التحويل/التحميل: {onnx_path}\")\n", " \n", " # اختبار الاستدلال ONNX\n", " import time\n", " img = Image.open('test_samples/english.png').convert('RGB')\n", " \n", " start = time.time()\n", " result = engine._recognize_trocr_onnx(img)\n", " elapsed = time.time() - start\n", " \n", " print(f\" ⏱️ وقت الاستدلال ONNX: {elapsed:.3f}s\")\n", " print(f\" الناتج: {result[:60] if result else '(فارغ)'}\")\n", " except Exception as e:\n", " print(f\" ⚠️ خطأ في ONNX: {e}\")\n", "except ImportError:\n", " print(\" ❌ ONNX Runtime غير مثبت\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 15: تشغيل واجهة Gradio للمعاينة التفاعلية" ] }, { "cell_type": "code", "metadata": { "id": "cell_15_gradio" }, "source": [ "# ============================================================\n", "# الخطوة 15: تشغيل واجهة Gradio\n", "# ============================================================\n", "\n", "print(\"🖥️ تشغيل واجهة Gradio...\")\n", "print(\"=\" * 50)\n", "\n", "try:\n", " import pyngrok\n", " from pyngrok import ngrok\n", " \n", " # فتح نفق\n", " public_url = ngrok.connect(7860)\n", " print(f\"🌐 رابط عام: {public_url}\")\n", " \n", " # تشغيل Streamlit\n", " !streamlit run app.py --server.port 8501 &\n", " \n", "except Exception as e:\n", " print(f\"❌ خطأ: {e}\")\n", " print(\"💡 يمكنك تشغيل الواجهة محلياً باستخدام: streamlit run app.py\")" ], "outputs": [], "execution_count": null }, { "cell_type": "markdown", "metadata": {}, "source": [ "## الخطوة 16: تنظيف الذاكرة وتحرير الموارد" ] }, { "cell_type": "code", "metadata": { "id": "cell_16_cleanup" }, "source": [ "# ============================================================\n", "# الخطوة 16: تنظيف الذاكرة\n", "# ============================================================\n", "\n", "import gc\n", "import torch\n", "\n", "print(\"🧹 تنظيف الذاكرة...\")\n", "\n", "# تحرير نماذج OCR\n", "try:\n", " engine.unload_models()\n", " print(\" ✅ تم تحرير نماذج OCR\")\n", "except:\n", " pass\n", "\n", "# تنظيف GPU\n", "if torch.cuda.is_available():\n", " torch.cuda.empty_cache()\n", " print(f\" ✅ GPU Cache محرر\")\n", "\n", "# تنظيف Python\n", "gc.collect()\n", "print(\" ✅ GC تم\")\n", "\n", "# ذاكرة نهائية\n", "import psutil\n", "mem = psutil.Process().memory_info()\n", "print(f\"\\n💾 الذاكرة النهائية: RSS={mem.rss/1e6:.1f} MB\")\n", "\n", "print(\"\\n✅ انتهى اختبار وتصحيح الأخطاء!\")\n", "print(\"=\" * 50)" ], "outputs": [], "execution_count": null } ] }