Spaces:
Running
feat: v5.0.0 — Comprehensive merge of 40 reference projects
Browse filesMajor new modules from project analysis:
- Active Learning: auto-retrain from user corrections with threshold monitoring
- Video OCR: extract timestamped text from video files (mp4/avi/mov/mkv)
- Data Augmentation: expand training data with rotation/noise/blur/brightness
- Protected Words: prevent spell correction of technical/medical/programming terms
- Study Guide Generator: Mermaid diagrams, Anki flashcards, HTML export
Enhanced Gradio HF App (10 tabs):
- Tab 8: Video OCR with frame interval control and timeline output
- Tab 9: Data Augmentation with 6 configurable parameters
- Updated docstring and tab numbering
This release integrates key features from 40 reference projects
including surya, PaddleOCR, docling, KITAB-Bench, and the
ocr_project_unified_v2 codebase.
- README.md +2 -2
- hf_app.py +271 -6
- modules/ai/__init__.py +4 -0
- modules/ai/active_learning.py +840 -0
- modules/nlp/__init__.py +6 -0
- modules/nlp/protected_words.py +653 -0
- modules/nlp/study_guide.py +1070 -0
- modules/vision/__init__.py +8 -0
- modules/vision/data_augmentation.py +829 -0
- modules/vision/video_ocr.py +597 -0
|
@@ -11,7 +11,7 @@ license: mit
|
|
| 11 |
|
| 12 |
<div align="center">
|
| 13 |
|
| 14 |
-
# 🧠 OmniFile AI Processor
|
| 15 |
|
| 16 |
**نظام ذكاء اصطناعي متكامل لمعالجة الملفات والنصوص والخط اليدوي**
|
| 17 |
**A Comprehensive AI System for File Processing, Text Analysis & Handwriting Recognition**
|
|
@@ -23,7 +23,7 @@ license: mit
|
|
| 23 |
[](https://github.com/DrAbdulmalek/OmniFile_Processor)
|
| 24 |
|
| 25 |
<p>
|
| 26 |
-
<b>Version:</b>
|
| 27 |
<b>Status:</b> ✅ CI-Verified
|
| 28 |
</p>
|
| 29 |
|
|
|
|
| 11 |
|
| 12 |
<div align="center">
|
| 13 |
|
| 14 |
+
# 🧠 OmniFile AI Processor v5.0.0
|
| 15 |
|
| 16 |
**نظام ذكاء اصطناعي متكامل لمعالجة الملفات والنصوص والخط اليدوي**
|
| 17 |
**A Comprehensive AI System for File Processing, Text Analysis & Handwriting Recognition**
|
|
|
|
| 23 |
[](https://github.com/DrAbdulmalek/OmniFile_Processor)
|
| 24 |
|
| 25 |
<p>
|
| 26 |
+
<b>Version:</b> v5.0.0 |
|
| 27 |
<b>Status:</b> ✅ CI-Verified
|
| 28 |
</p>
|
| 29 |
|
|
@@ -8,13 +8,15 @@ Multilingual handwriting OCR system supporting Arabic, English, and German.
|
|
| 8 |
|
| 9 |
Tabs:
|
| 10 |
1. OCR Processing — EasyOCR + TrOCR ensemble
|
| 11 |
-
2. Text Correction — ar-corrector + pyspellchecker
|
| 12 |
3. PDF Processing — PyMuPDF (fitz) page-by-page extraction
|
| 13 |
4. Translation — Helsinki-NLP MarianMT models + post-MT correction
|
| 14 |
5. Text Classification — Keyword-based multilingual categorization
|
| 15 |
6. Evaluation — CER / WER metrics (Levenshtein + jiwer)
|
| 16 |
7. Training Data — PDF to page/word/character crops for handwriting fine-tuning
|
| 17 |
-
8.
|
|
|
|
|
|
|
| 18 |
|
| 19 |
Author: Dr Abdulmalek Tamer Al-husseini
|
| 20 |
Email: Abdulmalek.husseini@gmail.com
|
|
@@ -1138,14 +1140,178 @@ ABOUT_MD = """
|
|
| 1138 |
"""
|
| 1139 |
|
| 1140 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1141 |
# ====================================================================
|
| 1142 |
# Build & Launch Gradio Application
|
| 1143 |
# ====================================================================
|
| 1144 |
|
| 1145 |
def build_app() -> gr.Blocks:
|
| 1146 |
"""
|
| 1147 |
-
Build the full Gradio Blocks application with
|
| 1148 |
-
/ بناء واجهة Gradio الكاملة مع
|
| 1149 |
"""
|
| 1150 |
|
| 1151 |
with gr.Blocks(
|
|
@@ -1495,8 +1661,107 @@ def build_app() -> gr.Blocks:
|
|
| 1495 |
)
|
| 1496 |
|
| 1497 |
# ================================================================
|
| 1498 |
-
# Tab 8 —
|
| 1499 |
-
# / التبويب الثامن:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1500 |
# ================================================================
|
| 1501 |
with gr.Tab("ℹ️ About"):
|
| 1502 |
dev = f"**Runtime**: `{DEVICE.upper()}`"
|
|
|
|
| 8 |
|
| 9 |
Tabs:
|
| 10 |
1. OCR Processing — EasyOCR + TrOCR ensemble
|
| 11 |
+
2. Text Correction — ar-corrector + pyspellchecker + protected words
|
| 12 |
3. PDF Processing — PyMuPDF (fitz) page-by-page extraction
|
| 13 |
4. Translation — Helsinki-NLP MarianMT models + post-MT correction
|
| 14 |
5. Text Classification — Keyword-based multilingual categorization
|
| 15 |
6. Evaluation — CER / WER metrics (Levenshtein + jiwer)
|
| 16 |
7. Training Data — PDF to page/word/character crops for handwriting fine-tuning
|
| 17 |
+
8. Video OCR — Extract text from video files with timestamps
|
| 18 |
+
9. Data Augmentation — Expand training data for Arabic handwriting
|
| 19 |
+
10. About — Project info, links, author
|
| 20 |
|
| 21 |
Author: Dr Abdulmalek Tamer Al-husseini
|
| 22 |
Email: Abdulmalek.husseini@gmail.com
|
|
|
|
| 1140 |
"""
|
| 1141 |
|
| 1142 |
|
| 1143 |
+
# ====================================================================
|
| 1144 |
+
# Tab 8 — Video OCR
|
| 1145 |
+
# / استخراج النصوص من مقاطع الفيديو
|
| 1146 |
+
# ====================================================================
|
| 1147 |
+
|
| 1148 |
+
def process_video_ocr(
|
| 1149 |
+
video_file,
|
| 1150 |
+
frame_interval: int = 30,
|
| 1151 |
+
progress=gr.Progress(),
|
| 1152 |
+
) -> str:
|
| 1153 |
+
"""
|
| 1154 |
+
Tab 8 handler — Extract text from video file.
|
| 1155 |
+
/ استخراج النصوص من ملف فيديو
|
| 1156 |
+
"""
|
| 1157 |
+
if video_file is None:
|
| 1158 |
+
return "⚠️ Please upload a video file."
|
| 1159 |
+
|
| 1160 |
+
progress(0.1, desc="Saving video file…")
|
| 1161 |
+
tmp_dir = tempfile.mkdtemp(prefix="omnifile_video_")
|
| 1162 |
+
video_path = os.path.join(tmp_dir, "input.mp4")
|
| 1163 |
+
shutil.copy2(video_file.name, video_path)
|
| 1164 |
+
|
| 1165 |
+
progress(0.2, desc="Initializing Video OCR…")
|
| 1166 |
+
try:
|
| 1167 |
+
from modules.vision.video_ocr import VideoOCR
|
| 1168 |
+
|
| 1169 |
+
vid_ocr = VideoOCR(frame_interval=frame_interval)
|
| 1170 |
+
|
| 1171 |
+
progress(0.3, desc="Extracting frames and running OCR…")
|
| 1172 |
+
timeline = vid_ocr.process_video(
|
| 1173 |
+
video_path=video_path,
|
| 1174 |
+
progress_callback=lambda cur, tot, msg: progress(0.3 + 0.6 * cur / max(tot, 1), desc=msg),
|
| 1175 |
+
)
|
| 1176 |
+
|
| 1177 |
+
if timeline.error:
|
| 1178 |
+
return f"❌ {timeline.error}"
|
| 1179 |
+
|
| 1180 |
+
progress(0.95, desc="Building results…")
|
| 1181 |
+
|
| 1182 |
+
# Build output
|
| 1183 |
+
out = "## 🎬 Video OCR Results\n\n"
|
| 1184 |
+
out += f"| Metric | Value |\n|--------|-------|\n"
|
| 1185 |
+
out += f"| **Total Frames Processed** | {timeline.total_frames} |\n"
|
| 1186 |
+
out += f"| **Frames with Text** | {timeline.frames_with_text} |\n"
|
| 1187 |
+
out += f"| **Average Confidence** | {timeline.avg_confidence:.1%} |\n"
|
| 1188 |
+
out += f"| **Frame Interval** | Every {frame_interval} frames |\n"
|
| 1189 |
+
out += f"| **Duration** | {timeline.total_frames / 30:.1f}s (estimated at 30fps) |\n\n"
|
| 1190 |
+
|
| 1191 |
+
if timeline.frames:
|
| 1192 |
+
out += "### 📝 Timeline\n\n"
|
| 1193 |
+
for frame in timeline.frames[:50]: # Limit display
|
| 1194 |
+
ts = frame.timestamp
|
| 1195 |
+
text = frame.text[:100] + ("..." if len(frame.text) > 100 else "")
|
| 1196 |
+
conf = frame.confidence
|
| 1197 |
+
out += f"- **[{ts}]** ({conf:.0%}) {text}\n"
|
| 1198 |
+
|
| 1199 |
+
if len(timeline.frames) > 50:
|
| 1200 |
+
out += f"\n*... and {len(timeline.frames) - 50} more frames*\n"
|
| 1201 |
+
|
| 1202 |
+
if timeline.full_text:
|
| 1203 |
+
out += "\n### 📄 Full Extracted Text\n\n"
|
| 1204 |
+
out += f"```\n{timeline.full_text[:2000]}\n```"
|
| 1205 |
+
if len(timeline.full_text) > 2000:
|
| 1206 |
+
out += "\n*(text truncated)*"
|
| 1207 |
+
|
| 1208 |
+
return out
|
| 1209 |
+
|
| 1210 |
+
except ImportError:
|
| 1211 |
+
return "❌ Video OCR module not available. Run with full OmniFile Processor installation."
|
| 1212 |
+
except Exception as e:
|
| 1213 |
+
logger.error("Video OCR error: %s", traceback.format_exc())
|
| 1214 |
+
return f"❌ Error: {e}"
|
| 1215 |
+
|
| 1216 |
+
|
| 1217 |
+
# ====================================================================
|
| 1218 |
+
# Tab 9 — Data Augmentation
|
| 1219 |
+
# / توسيع بيانات التدريب للخط العربي
|
| 1220 |
+
# ====================================================================
|
| 1221 |
+
|
| 1222 |
+
def augment_training_data(
|
| 1223 |
+
input_images,
|
| 1224 |
+
num_augmented: int = 5,
|
| 1225 |
+
rotation: float = 5.0,
|
| 1226 |
+
noise: float = 0.02,
|
| 1227 |
+
blur: bool = True,
|
| 1228 |
+
brightness: float = 0.2,
|
| 1229 |
+
progress=gr.Progress(),
|
| 1230 |
+
) -> Tuple[str, Optional[str]]:
|
| 1231 |
+
"""
|
| 1232 |
+
Tab 9 handler — Augment training images for better model training.
|
| 1233 |
+
/ توسيع صور التدريب لتحسين أداء النموذج
|
| 1234 |
+
"""
|
| 1235 |
+
if input_images is None:
|
| 1236 |
+
return "⚠️ Please upload training images.", None
|
| 1237 |
+
|
| 1238 |
+
progress(0.1, desc="Setting up augmentation…")
|
| 1239 |
+
tmp_dir = tempfile.mkdtemp(prefix="omnifile_aug_")
|
| 1240 |
+
|
| 1241 |
+
try:
|
| 1242 |
+
from modules.vision.data_augmentation import DataAugmentor
|
| 1243 |
+
|
| 1244 |
+
# Save uploaded images
|
| 1245 |
+
image_paths = []
|
| 1246 |
+
for img_info in input_images:
|
| 1247 |
+
img_path = os.path.join(tmp_dir, os.path.basename(img_info.name))
|
| 1248 |
+
shutil.copy2(img_info.name, img_path)
|
| 1249 |
+
image_paths.append(img_path)
|
| 1250 |
+
|
| 1251 |
+
progress(0.2, desc="Processing images…")
|
| 1252 |
+
augmentor = DataAugmentor(
|
| 1253 |
+
rotation_max=rotation,
|
| 1254 |
+
noise_std=noise,
|
| 1255 |
+
blur_kernel=5 if blur else 0,
|
| 1256 |
+
brightness_range=brightness,
|
| 1257 |
+
)
|
| 1258 |
+
|
| 1259 |
+
progress(0.3, desc="Generating augmented images…")
|
| 1260 |
+
|
| 1261 |
+
results = []
|
| 1262 |
+
for i, img_path in enumerate(image_paths):
|
| 1263 |
+
aug_results = augmentor.augment_image(
|
| 1264 |
+
img_path,
|
| 1265 |
+
num_augmentations=num_augmented,
|
| 1266 |
+
output_dir=os.path.join(tmp_dir, "augmented"),
|
| 1267 |
+
)
|
| 1268 |
+
results.extend(aug_results)
|
| 1269 |
+
progress(0.3 + 0.5 * (i + 1) / len(image_paths),
|
| 1270 |
+
desc=f"Image {i+1}/{len(image_paths)}…")
|
| 1271 |
+
|
| 1272 |
+
progress(0.85, desc="Packaging results…")
|
| 1273 |
+
|
| 1274 |
+
# Create zip
|
| 1275 |
+
import zipfile
|
| 1276 |
+
zip_path = os.path.join(tmp_dir, "augmented_data.zip")
|
| 1277 |
+
aug_dir = os.path.join(tmp_dir, "augmented")
|
| 1278 |
+
if os.path.exists(aug_dir):
|
| 1279 |
+
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
| 1280 |
+
for root, dirs, files in os.walk(aug_dir):
|
| 1281 |
+
for fname in files:
|
| 1282 |
+
fpath = os.path.join(root, fname)
|
| 1283 |
+
zf.write(fpath, os.path.relpath(fpath, aug_dir))
|
| 1284 |
+
|
| 1285 |
+
progress(1.0, desc="Done!")
|
| 1286 |
+
|
| 1287 |
+
out = "## 🔄 Data Augmentation Complete!\n\n"
|
| 1288 |
+
out += f"| Metric | Value |\n|--------|-------|\n"
|
| 1289 |
+
out += f"| **Original Images** | {len(image_paths)} |\n"
|
| 1290 |
+
out += f"| **Augmented Images** | {len(results)} |\n"
|
| 1291 |
+
out += f"| **Multiplication Factor** | {num_augmented}x |\n"
|
| 1292 |
+
out += f"| **Rotation Range** | ±{rotation}° |\n"
|
| 1293 |
+
out += f"| **Noise Level** | {noise} |\n"
|
| 1294 |
+
out += f"| **Blur** | {'Enabled' if blur else 'Disabled'} |\n"
|
| 1295 |
+
out += f"| **Brightness Range** | ±{brightness} |\n\n"
|
| 1296 |
+
out += "📥 Download the ZIP file containing augmented images."
|
| 1297 |
+
|
| 1298 |
+
return out, zip_path if os.path.exists(zip_path) else None
|
| 1299 |
+
|
| 1300 |
+
except ImportError:
|
| 1301 |
+
return "❌ Data Augmentation module not available.", None
|
| 1302 |
+
except Exception as e:
|
| 1303 |
+
logger.error("Augmentation error: %s", traceback.format_exc())
|
| 1304 |
+
return f"❌ Error: {e}", None
|
| 1305 |
+
|
| 1306 |
+
|
| 1307 |
# ====================================================================
|
| 1308 |
# Build & Launch Gradio Application
|
| 1309 |
# ====================================================================
|
| 1310 |
|
| 1311 |
def build_app() -> gr.Blocks:
|
| 1312 |
"""
|
| 1313 |
+
Build the full Gradio Blocks application with 10 tabs.
|
| 1314 |
+
/ بناء واجهة Gradio الكاملة مع 10 تبويبات
|
| 1315 |
"""
|
| 1316 |
|
| 1317 |
with gr.Blocks(
|
|
|
|
| 1661 |
)
|
| 1662 |
|
| 1663 |
# ================================================================
|
| 1664 |
+
# Tab 8 — Video OCR
|
| 1665 |
+
# / التبويب الثامن: استخراج النصوص من الفيديو
|
| 1666 |
+
# ================================================================
|
| 1667 |
+
with gr.Tab("🎬 Video OCR"):
|
| 1668 |
+
gr.Markdown(
|
| 1669 |
+
"### Extract Text from Video Files "
|
| 1670 |
+
"### استخراج النصوص من مقاطع الفيديو"
|
| 1671 |
+
)
|
| 1672 |
+
gr.Markdown(
|
| 1673 |
+
"📤 Upload a video → Get timestamped text from frames "
|
| 1674 |
+
"Supports MP4, AVI, MOV, MKV and more."
|
| 1675 |
+
)
|
| 1676 |
+
|
| 1677 |
+
with gr.Row():
|
| 1678 |
+
with gr.Column(scale=1):
|
| 1679 |
+
video_file = gr.File(
|
| 1680 |
+
label="📤 Upload Video / رفع فيديو",
|
| 1681 |
+
file_types=[".mp4", ".avi", ".mov", ".mkv", ".webm"],
|
| 1682 |
+
interactive=True,
|
| 1683 |
+
)
|
| 1684 |
+
video_interval = gr.Slider(
|
| 1685 |
+
label="⏱️ Frame Interval / فاصل الإطارات",
|
| 1686 |
+
minimum=5, maximum=120, value=30, step=5,
|
| 1687 |
+
info="Extract text every N frames (30 = ~1 sec at 30fps)",
|
| 1688 |
+
)
|
| 1689 |
+
video_btn = gr.Button(
|
| 1690 |
+
"🎬 Extract Text / استخراج النص",
|
| 1691 |
+
variant="primary", size="lg",
|
| 1692 |
+
)
|
| 1693 |
+
|
| 1694 |
+
with gr.Column(scale=2):
|
| 1695 |
+
video_output = gr.Markdown("")
|
| 1696 |
+
|
| 1697 |
+
video_btn.click(
|
| 1698 |
+
fn=process_video_ocr,
|
| 1699 |
+
inputs=[video_file, video_interval],
|
| 1700 |
+
outputs=[video_output],
|
| 1701 |
+
)
|
| 1702 |
+
|
| 1703 |
+
# ================================================================
|
| 1704 |
+
# Tab 9 — Data Augmentation
|
| 1705 |
+
# / التبويب التاسع: توسيع بيانات التدريب
|
| 1706 |
+
# ================================================================
|
| 1707 |
+
with gr.Tab("🔄 Data Augmentation"):
|
| 1708 |
+
gr.Markdown(
|
| 1709 |
+
"### Augment Training Images for Better Handwriting Recognition "
|
| 1710 |
+
"### توسيع صور التدريب لتحسين التعرف على الخط اليدوي"
|
| 1711 |
+
)
|
| 1712 |
+
gr.Markdown(
|
| 1713 |
+
"📤 Upload training images → Get augmented copies with rotation, noise, blur, and brightness variations."
|
| 1714 |
+
)
|
| 1715 |
+
|
| 1716 |
+
with gr.Row():
|
| 1717 |
+
with gr.Column(scale=1):
|
| 1718 |
+
aug_files = gr.File(
|
| 1719 |
+
label="📤 Upload Images / رفع صور",
|
| 1720 |
+
file_types=[".png", ".jpg", ".jpeg", ".bmp", ".webp"],
|
| 1721 |
+
interactive=True,
|
| 1722 |
+
multiple=True,
|
| 1723 |
+
)
|
| 1724 |
+
aug_count = gr.Slider(
|
| 1725 |
+
label="🔢 Augmentations per Image / عدد النسخ لكل صورة",
|
| 1726 |
+
minimum=1, maximum=20, value=5, step=1,
|
| 1727 |
+
)
|
| 1728 |
+
aug_rotation = gr.Slider(
|
| 1729 |
+
label="🔄 Rotation (degrees) / الدوران",
|
| 1730 |
+
minimum=0, maximum=15, value=5, step=1,
|
| 1731 |
+
)
|
| 1732 |
+
aug_noise = gr.Slider(
|
| 1733 |
+
label="📡 Noise Level / مستوى الضوضاء",
|
| 1734 |
+
minimum=0, maximum=0.1, value=0.02, step=0.01,
|
| 1735 |
+
)
|
| 1736 |
+
aug_blur = gr.Checkbox(
|
| 1737 |
+
label="🌫️ Enable Blur / تفعيل الضبابية",
|
| 1738 |
+
value=True,
|
| 1739 |
+
)
|
| 1740 |
+
aug_brightness = gr.Slider(
|
| 1741 |
+
label="💡 Brightness Range / نطاق السطوع",
|
| 1742 |
+
minimum=0, maximum=0.5, value=0.2, step=0.05,
|
| 1743 |
+
)
|
| 1744 |
+
aug_btn = gr.Button(
|
| 1745 |
+
"🔄 Augment Images / توسيع الصور",
|
| 1746 |
+
variant="primary", size="lg",
|
| 1747 |
+
)
|
| 1748 |
+
|
| 1749 |
+
with gr.Column(scale=2):
|
| 1750 |
+
aug_output = gr.Markdown("")
|
| 1751 |
+
aug_download = gr.File(
|
| 1752 |
+
label="📥 Download Augmented Data / تنزيل البيانات الموسعة",
|
| 1753 |
+
interactive=False,
|
| 1754 |
+
)
|
| 1755 |
+
|
| 1756 |
+
aug_btn.click(
|
| 1757 |
+
fn=augment_training_data,
|
| 1758 |
+
inputs=[aug_files, aug_count, aug_rotation, aug_noise, aug_blur, aug_brightness],
|
| 1759 |
+
outputs=[aug_output, aug_download],
|
| 1760 |
+
)
|
| 1761 |
+
|
| 1762 |
+
# ================================================================
|
| 1763 |
+
# Tab 10 — About
|
| 1764 |
+
# / التبويب العاشر: حول المشروع
|
| 1765 |
# ================================================================
|
| 1766 |
with gr.Tab("ℹ️ About"):
|
| 1767 |
dev = f"**Runtime**: `{DEVICE.upper()}`"
|
|
@@ -5,14 +5,18 @@ OmniFile AI Processor — AI Module (Self-Learning & AI Refinement)
|
|
| 5 |
- مطابقة الأنماط البصرية باستخدام SSIM (Pattern Matching)
|
| 6 |
- قاعدة بيانات التصحيحات الذاتية (Pattern Database)
|
| 7 |
- تحسين النصوص عبر Google Gemini API (Gemini Refiner)
|
|
|
|
| 8 |
"""
|
| 9 |
from modules.ai.pattern_db import PatternDatabase
|
| 10 |
from modules.ai.pattern_matcher import PatternMatcher, PatternMatch
|
| 11 |
from modules.ai.gemini_refiner import GeminiRefiner
|
|
|
|
| 12 |
|
| 13 |
__all__ = [
|
| 14 |
"PatternDatabase",
|
| 15 |
"PatternMatcher",
|
| 16 |
"PatternMatch",
|
| 17 |
"GeminiRefiner",
|
|
|
|
|
|
|
| 18 |
]
|
|
|
|
| 5 |
- مطابقة الأنماط البصرية باستخدام SSIM (Pattern Matching)
|
| 6 |
- قاعدة بيانات التصحيحات الذاتية (Pattern Database)
|
| 7 |
- تحسين النصوص عبر Google Gemini API (Gemini Refiner)
|
| 8 |
+
- التعلم النشط وتحسين النماذج عبر تصحيحات المستخدم (Active Learning)
|
| 9 |
"""
|
| 10 |
from modules.ai.pattern_db import PatternDatabase
|
| 11 |
from modules.ai.pattern_matcher import PatternMatcher, PatternMatch
|
| 12 |
from modules.ai.gemini_refiner import GeminiRefiner
|
| 13 |
+
from modules.ai.active_learning import ActiveLearningDB, ActiveLearner
|
| 14 |
|
| 15 |
__all__ = [
|
| 16 |
"PatternDatabase",
|
| 17 |
"PatternMatcher",
|
| 18 |
"PatternMatch",
|
| 19 |
"GeminiRefiner",
|
| 20 |
+
"ActiveLearningDB",
|
| 21 |
+
"ActiveLearner",
|
| 22 |
]
|
|
@@ -0,0 +1,840 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
التعلم النشط — Active Learning Module
|
| 3 |
+
========================================
|
| 4 |
+
نظام تعلم نشط يحسّن دقة OCR عبر تصحيحات المستخدم وإعادة التدريب التلقائي.
|
| 5 |
+
|
| 6 |
+
القدرات:
|
| 7 |
+
- تخزين تصحيحات المستخدم في قاعدة بيانات SQLite مع إلغاء التكرار
|
| 8 |
+
- مراقبة وصول التصحيحات لعتبة إعادة التدريب
|
| 9 |
+
- تصدير بيانات التدريب بصيغة JSONL متوافقة مع TrOCR
|
| 10 |
+
- تتبع إصدارات النماذج بعد الضبط الدقيق
|
| 11 |
+
- إحصائيات شاملة وتحليلات تحسين الثقة
|
| 12 |
+
|
| 13 |
+
مثال الاستخدام:
|
| 14 |
+
>>> learner = ActiveLearner(db_path="data/active_learning.db")
|
| 15 |
+
>>> learner.log_correction("hellp", "hello", confidence=0.45, source="trocr")
|
| 16 |
+
>>> learner.log_correction("wrld", "world", confidence=0.62, source="easyocr")
|
| 17 |
+
>>> stats = learner.get_stats()
|
| 18 |
+
>>> if learner.should_retrain():
|
| 19 |
+
... path = learner.export_training_data("training_data.jsonl")
|
| 20 |
+
... learner.record_model_version("trocr-v2", metrics={"cer": 0.03})
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import json
|
| 26 |
+
import logging
|
| 27 |
+
import sqlite3
|
| 28 |
+
from datetime import datetime
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
from typing import Any, Optional
|
| 31 |
+
|
| 32 |
+
logger = logging.getLogger(__name__)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# ======================================================================
|
| 36 |
+
# قاعدة بيانات التعلم النشط
|
| 37 |
+
# ======================================================================
|
| 38 |
+
|
| 39 |
+
class ActiveLearningDB:
|
| 40 |
+
"""قاعدة بيانات SQLite لإدارة نظام التعلم النشط.
|
| 41 |
+
|
| 42 |
+
تدير ثلاثة جداول:
|
| 43 |
+
- **corrections**: تصحيحات المستخدم (النص الأصلي ← النص المصحّح)
|
| 44 |
+
- **training_data**: بيانات التدريب المُصدَّرة للضبط الدقيق
|
| 45 |
+
- **fine_tuned_models**: إصدارات النماذج ونتائج التقييم
|
| 46 |
+
|
| 47 |
+
Attributes:
|
| 48 |
+
db_path: مسار ملف قاعدة البيانات.
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
def __init__(self, db_path: str | Path = "data/active_learning.db") -> None:
|
| 52 |
+
"""تهيئة قاعدة بيانات التعلم النشط.
|
| 53 |
+
|
| 54 |
+
Args:
|
| 55 |
+
db_path: مسار ملف SQLite (يُنشأ تلقائياً إذا لم يوجد).
|
| 56 |
+
"""
|
| 57 |
+
self.db_path = Path(db_path)
|
| 58 |
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
| 59 |
+
self._connection: Optional[sqlite3.Connection] = None
|
| 60 |
+
self._initialize_database()
|
| 61 |
+
|
| 62 |
+
# ------------------------------------------------------------------
|
| 63 |
+
# إدارة الاتصال
|
| 64 |
+
# ------------------------------------------------------------------
|
| 65 |
+
|
| 66 |
+
def _get_connection(self) -> sqlite3.Connection:
|
| 67 |
+
"""الحصول على اتصال نشط بقاعدة البيانات (يُنشأ عند أول استخدام).
|
| 68 |
+
|
| 69 |
+
Returns:
|
| 70 |
+
كائن sqlite3.Connection مُهيأ.
|
| 71 |
+
"""
|
| 72 |
+
if self._connection is None:
|
| 73 |
+
self._connection = sqlite3.connect(
|
| 74 |
+
str(self.db_path),
|
| 75 |
+
check_same_thread=False,
|
| 76 |
+
)
|
| 77 |
+
self._connection.row_factory = sqlite3.Row
|
| 78 |
+
self._connection.execute("PRAGMA journal_mode=WAL")
|
| 79 |
+
return self._connection
|
| 80 |
+
|
| 81 |
+
def _initialize_database(self) -> None:
|
| 82 |
+
"""إنشاء الجداول إذا لم تكن موجودة.
|
| 83 |
+
|
| 84 |
+
الجداول المُنشأة:
|
| 85 |
+
- corrections: تخزين تصحيحات المستخدم مع إلغاء التكرار
|
| 86 |
+
- training_data: بيانات التدريب بتنسيق JSONL
|
| 87 |
+
- fine_tuned_models: سجل إصدارات النماذج المُضبَّطة
|
| 88 |
+
"""
|
| 89 |
+
conn = self._get_connection()
|
| 90 |
+
cursor = conn.cursor()
|
| 91 |
+
|
| 92 |
+
cursor.executescript("""
|
| 93 |
+
CREATE TABLE IF NOT EXISTS corrections (
|
| 94 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 95 |
+
original_text TEXT NOT NULL,
|
| 96 |
+
corrected_text TEXT NOT NULL,
|
| 97 |
+
original_confidence REAL DEFAULT 0.0,
|
| 98 |
+
source_engine TEXT DEFAULT '',
|
| 99 |
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
| 100 |
+
dedup_hash TEXT NOT NULL,
|
| 101 |
+
UNIQUE(dedup_hash)
|
| 102 |
+
);
|
| 103 |
+
|
| 104 |
+
CREATE INDEX IF NOT EXISTS idx_corrections_source
|
| 105 |
+
ON corrections(source_engine);
|
| 106 |
+
|
| 107 |
+
CREATE INDEX IF NOT EXISTS idx_corrections_confidence
|
| 108 |
+
ON corrections(original_confidence);
|
| 109 |
+
|
| 110 |
+
CREATE TABLE IF NOT EXISTS training_data (
|
| 111 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 112 |
+
correction_id INTEGER NOT NULL,
|
| 113 |
+
original_text TEXT NOT NULL,
|
| 114 |
+
corrected_text TEXT NOT NULL,
|
| 115 |
+
image_data BLOB,
|
| 116 |
+
source_engine TEXT DEFAULT '',
|
| 117 |
+
exported_at TEXT NOT NULL DEFAULT (datetime('now')),
|
| 118 |
+
included_in_version TEXT DEFAULT NULL,
|
| 119 |
+
FOREIGN KEY (correction_id) REFERENCES corrections(id)
|
| 120 |
+
);
|
| 121 |
+
|
| 122 |
+
CREATE INDEX IF NOT EXISTS idx_training_version
|
| 123 |
+
ON training_data(included_in_version);
|
| 124 |
+
|
| 125 |
+
CREATE TABLE IF NOT EXISTS fine_tuned_models (
|
| 126 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 127 |
+
version_name TEXT NOT NULL UNIQUE,
|
| 128 |
+
base_model TEXT DEFAULT '',
|
| 129 |
+
training_samples INTEGER DEFAULT 0,
|
| 130 |
+
metrics_json TEXT DEFAULT '{}',
|
| 131 |
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
| 132 |
+
notes TEXT DEFAULT ''
|
| 133 |
+
);
|
| 134 |
+
""")
|
| 135 |
+
|
| 136 |
+
conn.commit()
|
| 137 |
+
logger.debug("تم تهيئة قاعدة بيانات التعلم النشط: %s", self.db_path)
|
| 138 |
+
|
| 139 |
+
# ------------------------------------------------------------------
|
| 140 |
+
# عمليات التصحيحات
|
| 141 |
+
# ------------------------------------------------------------------
|
| 142 |
+
|
| 143 |
+
def add_correction(
|
| 144 |
+
self,
|
| 145 |
+
original_text: str,
|
| 146 |
+
corrected_text: str,
|
| 147 |
+
original_confidence: float = 0.0,
|
| 148 |
+
source_engine: str = "",
|
| 149 |
+
) -> Optional[int]:
|
| 150 |
+
"""إضافة تصحيح جديد مع إلغاء التكرار التلقائي.
|
| 151 |
+
|
| 152 |
+
يُحسب تجزئة فريدة (hash) من النص الأصلي والمُصحَّح لمنع التكرار.
|
| 153 |
+
|
| 154 |
+
Args:
|
| 155 |
+
original_text: النص الأصلي (قبل التصحيح).
|
| 156 |
+
corrected_text: النص المصحّح من المستخدم.
|
| 157 |
+
original_confidence: ثقة OCR الأصلية (0.0 - 1.0).
|
| 158 |
+
source_engine: محرك OCR المصدر.
|
| 159 |
+
|
| 160 |
+
Returns:
|
| 161 |
+
معرّف السجل (ID) أو None إذا كان مكرراً.
|
| 162 |
+
"""
|
| 163 |
+
import hashlib
|
| 164 |
+
|
| 165 |
+
conn = self._get_connection()
|
| 166 |
+
cursor = conn.cursor()
|
| 167 |
+
|
| 168 |
+
# حساب تجزئة فريدة لإلغاء التكرار
|
| 169 |
+
dedup_key = f"{original_text.strip().lower()}|||{corrected_text.strip()}"
|
| 170 |
+
dedup_hash = hashlib.sha256(dedup_key.encode("utf-8")).hexdigest()
|
| 171 |
+
|
| 172 |
+
try:
|
| 173 |
+
cursor.execute(
|
| 174 |
+
"""INSERT INTO corrections
|
| 175 |
+
(original_text, corrected_text, original_confidence,
|
| 176 |
+
source_engine, dedup_hash)
|
| 177 |
+
VALUES (?, ?, ?, ?, ?)""",
|
| 178 |
+
(
|
| 179 |
+
original_text.strip(),
|
| 180 |
+
corrected_text.strip(),
|
| 181 |
+
original_confidence,
|
| 182 |
+
source_engine,
|
| 183 |
+
dedup_hash,
|
| 184 |
+
),
|
| 185 |
+
)
|
| 186 |
+
conn.commit()
|
| 187 |
+
row_id = cursor.lastrowid
|
| 188 |
+
logger.info(
|
| 189 |
+
"تصحيح جديد #%d: '%s' → '%s' (ثقة: %.2f, محرك: %s)",
|
| 190 |
+
row_id, original_text, corrected_text,
|
| 191 |
+
original_confidence, source_engine,
|
| 192 |
+
)
|
| 193 |
+
return row_id
|
| 194 |
+
except sqlite3.IntegrityError:
|
| 195 |
+
logger.debug(
|
| 196 |
+
"تصحيح مكرر (تم التجاهل): '%s' → '%s'",
|
| 197 |
+
original_text, corrected_text,
|
| 198 |
+
)
|
| 199 |
+
return None
|
| 200 |
+
|
| 201 |
+
def get_corrections(
|
| 202 |
+
self,
|
| 203 |
+
min_confidence: float = 0.0,
|
| 204 |
+
source_engine: Optional[str] = None,
|
| 205 |
+
limit: int = 1000,
|
| 206 |
+
) -> list[dict[str, Any]]:
|
| 207 |
+
"""استرجاع التصحيحات مع فلاتر اختيارية.
|
| 208 |
+
|
| 209 |
+
Args:
|
| 210 |
+
min_confidence: أقل حد للثقة (للتركيز على التصحيحات ذات الثقة المنخفضة).
|
| 211 |
+
source_engine: فلتر حسب المحرك.
|
| 212 |
+
limit: أقصى عدد نتائج.
|
| 213 |
+
|
| 214 |
+
Returns:
|
| 215 |
+
قائمة قواميس التصحيحات.
|
| 216 |
+
"""
|
| 217 |
+
conn = self._get_connection()
|
| 218 |
+
cursor = conn.cursor()
|
| 219 |
+
|
| 220 |
+
query = """
|
| 221 |
+
SELECT id, original_text, corrected_text, original_confidence,
|
| 222 |
+
source_engine, created_at, dedup_hash
|
| 223 |
+
FROM corrections
|
| 224 |
+
WHERE original_confidence >= ?
|
| 225 |
+
"""
|
| 226 |
+
params: list[Any] = [min_confidence]
|
| 227 |
+
|
| 228 |
+
if source_engine:
|
| 229 |
+
query += " AND source_engine = ?"
|
| 230 |
+
params.append(source_engine)
|
| 231 |
+
|
| 232 |
+
query += " ORDER BY original_confidence ASC, created_at DESC LIMIT ?"
|
| 233 |
+
params.append(limit)
|
| 234 |
+
|
| 235 |
+
cursor.execute(query, params)
|
| 236 |
+
|
| 237 |
+
return [
|
| 238 |
+
{
|
| 239 |
+
"id": row["id"],
|
| 240 |
+
"original_text": row["original_text"],
|
| 241 |
+
"corrected_text": row["corrected_text"],
|
| 242 |
+
"original_confidence": row["original_confidence"],
|
| 243 |
+
"source_engine": row["source_engine"],
|
| 244 |
+
"created_at": row["created_at"],
|
| 245 |
+
}
|
| 246 |
+
for row in cursor.fetchall()
|
| 247 |
+
]
|
| 248 |
+
|
| 249 |
+
def count_corrections(self) -> int:
|
| 250 |
+
"""عدد التصحيحات الفريدة.
|
| 251 |
+
|
| 252 |
+
Returns:
|
| 253 |
+
عدد السجلات في جدول corrections.
|
| 254 |
+
"""
|
| 255 |
+
conn = self._get_connection()
|
| 256 |
+
cursor = conn.cursor()
|
| 257 |
+
cursor.execute("SELECT COUNT(*) as cnt FROM corrections")
|
| 258 |
+
return cursor.fetchone()["cnt"]
|
| 259 |
+
|
| 260 |
+
# ------------------------------------------------------------------
|
| 261 |
+
# عمليات بيانات التدريب
|
| 262 |
+
# ------------------------------------------------------------------
|
| 263 |
+
|
| 264 |
+
def add_training_entry(
|
| 265 |
+
self,
|
| 266 |
+
correction_id: int,
|
| 267 |
+
original_text: str,
|
| 268 |
+
corrected_text: str,
|
| 269 |
+
image_data: Optional[bytes] = None,
|
| 270 |
+
source_engine: str = "",
|
| 271 |
+
) -> int:
|
| 272 |
+
"""إضافة سجل بيانات تدريب مرتبط بتصحيح.
|
| 273 |
+
|
| 274 |
+
Args:
|
| 275 |
+
correction_id: معرّف التصحيح المرتبط.
|
| 276 |
+
original_text: النص الأصلي.
|
| 277 |
+
corrected_text: النص المصحّح.
|
| 278 |
+
image_data: بيانات الصورة (اختياري).
|
| 279 |
+
source_engine: محرك OCR المصدر.
|
| 280 |
+
|
| 281 |
+
Returns:
|
| 282 |
+
معرّف سجل التدريب.
|
| 283 |
+
"""
|
| 284 |
+
conn = self._get_connection()
|
| 285 |
+
cursor = conn.cursor()
|
| 286 |
+
|
| 287 |
+
cursor.execute(
|
| 288 |
+
"""INSERT INTO training_data
|
| 289 |
+
(correction_id, original_text, corrected_text,
|
| 290 |
+
image_data, source_engine)
|
| 291 |
+
VALUES (?, ?, ?, ?, ?)""",
|
| 292 |
+
(correction_id, original_text, corrected_text,
|
| 293 |
+
image_data, source_engine),
|
| 294 |
+
)
|
| 295 |
+
conn.commit()
|
| 296 |
+
return cursor.lastrowid # type: ignore[return-value]
|
| 297 |
+
|
| 298 |
+
def get_training_data(
|
| 299 |
+
self,
|
| 300 |
+
included_in_version: Optional[str] = None,
|
| 301 |
+
limit: int = 5000,
|
| 302 |
+
) -> list[dict[str, Any]]:
|
| 303 |
+
"""استرجاع بيانات التدريب.
|
| 304 |
+
|
| 305 |
+
Args:
|
| 306 |
+
included_in_version: فلتر حسب إصدار النموذج.
|
| 307 |
+
limit: أقصى عدد نتائج.
|
| 308 |
+
|
| 309 |
+
Returns:
|
| 310 |
+
قائمة بيانات التدريب.
|
| 311 |
+
"""
|
| 312 |
+
conn = self._get_connection()
|
| 313 |
+
cursor = conn.cursor()
|
| 314 |
+
|
| 315 |
+
if included_in_version is not None:
|
| 316 |
+
cursor.execute(
|
| 317 |
+
"""SELECT id, correction_id, original_text, corrected_text,
|
| 318 |
+
source_engine, exported_at, included_in_version
|
| 319 |
+
FROM training_data
|
| 320 |
+
WHERE included_in_version = ?
|
| 321 |
+
ORDER BY id DESC LIMIT ?""",
|
| 322 |
+
(included_in_version, limit),
|
| 323 |
+
)
|
| 324 |
+
else:
|
| 325 |
+
cursor.execute(
|
| 326 |
+
"""SELECT id, correction_id, original_text, corrected_text,
|
| 327 |
+
source_engine, exported_at, included_in_version
|
| 328 |
+
FROM training_data
|
| 329 |
+
WHERE included_in_version IS NULL
|
| 330 |
+
ORDER BY id DESC LIMIT ?""",
|
| 331 |
+
(limit,),
|
| 332 |
+
)
|
| 333 |
+
|
| 334 |
+
return [
|
| 335 |
+
{
|
| 336 |
+
"id": row["id"],
|
| 337 |
+
"correction_id": row["correction_id"],
|
| 338 |
+
"original_text": row["original_text"],
|
| 339 |
+
"corrected_text": row["corrected_text"],
|
| 340 |
+
"source_engine": row["source_engine"],
|
| 341 |
+
"exported_at": row["exported_at"],
|
| 342 |
+
"included_in_version": row["included_in_version"],
|
| 343 |
+
}
|
| 344 |
+
for row in cursor.fetchall()
|
| 345 |
+
]
|
| 346 |
+
|
| 347 |
+
def mark_training_used(self, version_name: str) -> int:
|
| 348 |
+
"""تحديد بيانات التدريب غير المستخدمة كـمُستخدمة في إصدار محدد.
|
| 349 |
+
|
| 350 |
+
Args:
|
| 351 |
+
version_name: اسم إصدار النموذج.
|
| 352 |
+
|
| 353 |
+
Returns:
|
| 354 |
+
عدد السجلات المُحدَّثة.
|
| 355 |
+
"""
|
| 356 |
+
conn = self._get_connection()
|
| 357 |
+
cursor = conn.cursor()
|
| 358 |
+
|
| 359 |
+
cursor.execute(
|
| 360 |
+
"""UPDATE training_data
|
| 361 |
+
SET included_in_version = ?
|
| 362 |
+
WHERE included_in_version IS NULL""",
|
| 363 |
+
(version_name,),
|
| 364 |
+
)
|
| 365 |
+
conn.commit()
|
| 366 |
+
updated = cursor.rowcount
|
| 367 |
+
logger.info("تم تحديد %d سجل تدريب كـمُستخدم في الإصدار '%s'", updated, version_name)
|
| 368 |
+
return updated
|
| 369 |
+
|
| 370 |
+
# ------------------------------------------------------------------
|
| 371 |
+
# عمليات النماذج
|
| 372 |
+
# ------------------------------------------------------------------
|
| 373 |
+
|
| 374 |
+
def add_model_version(
|
| 375 |
+
self,
|
| 376 |
+
version_name: str,
|
| 377 |
+
base_model: str = "",
|
| 378 |
+
training_samples: int = 0,
|
| 379 |
+
metrics: Optional[dict[str, Any]] = None,
|
| 380 |
+
notes: str = "",
|
| 381 |
+
) -> int:
|
| 382 |
+
"""تسجيل إصدار نموذج جديد بعد الضبط الدقيق.
|
| 383 |
+
|
| 384 |
+
Args:
|
| 385 |
+
version_name: اسم الإصدار (يجب أن يكون فريداً).
|
| 386 |
+
base_model: اسم النموذج الأساسي.
|
| 387 |
+
training_samples: عدد عيّنات التدريب المستخدمة.
|
| 388 |
+
metrics: قاموس مقاييس الأداء (مثل CER, WER, accuracy).
|
| 389 |
+
notes: ملاحظات إضافية.
|
| 390 |
+
|
| 391 |
+
Returns:
|
| 392 |
+
معرّف الإصدار.
|
| 393 |
+
|
| 394 |
+
Raises:
|
| 395 |
+
ValueError: إذا كان اسم الإصدار مكرراً.
|
| 396 |
+
"""
|
| 397 |
+
conn = self._get_connection()
|
| 398 |
+
cursor = conn.cursor()
|
| 399 |
+
|
| 400 |
+
metrics_json = json.dumps(metrics or {}, ensure_ascii=False)
|
| 401 |
+
|
| 402 |
+
try:
|
| 403 |
+
cursor.execute(
|
| 404 |
+
"""INSERT INTO fine_tuned_models
|
| 405 |
+
(version_name, base_model, training_samples,
|
| 406 |
+
metrics_json, notes)
|
| 407 |
+
VALUES (?, ?, ?, ?, ?)""",
|
| 408 |
+
(version_name, base_model, training_samples,
|
| 409 |
+
metrics_json, notes),
|
| 410 |
+
)
|
| 411 |
+
conn.commit()
|
| 412 |
+
model_id = cursor.lastrowid
|
| 413 |
+
logger.info(
|
| 414 |
+
"إصدار نموذج جديد #%d: '%s' (عدد عيّنات: %d)",
|
| 415 |
+
model_id, version_name, training_samples,
|
| 416 |
+
)
|
| 417 |
+
return model_id # type: ignore[return-value]
|
| 418 |
+
except sqlite3.IntegrityError:
|
| 419 |
+
raise ValueError(f"اسم الإصدار مكرر: '{version_name}'")
|
| 420 |
+
|
| 421 |
+
def get_model_versions(self, limit: int = 20) -> list[dict[str, Any]]:
|
| 422 |
+
"""استرجاع سجل إصدارات النماذج.
|
| 423 |
+
|
| 424 |
+
Args:
|
| 425 |
+
limit: أقصى عدد نتائج.
|
| 426 |
+
|
| 427 |
+
Returns:
|
| 428 |
+
قائمة إصدارات النماذج مع مقاييس الأداء.
|
| 429 |
+
"""
|
| 430 |
+
conn = self._get_connection()
|
| 431 |
+
cursor = conn.cursor()
|
| 432 |
+
|
| 433 |
+
cursor.execute(
|
| 434 |
+
"""SELECT id, version_name, base_model, training_samples,
|
| 435 |
+
metrics_json, created_at, notes
|
| 436 |
+
FROM fine_tuned_models
|
| 437 |
+
ORDER BY created_at DESC LIMIT ?""",
|
| 438 |
+
(limit,),
|
| 439 |
+
)
|
| 440 |
+
|
| 441 |
+
versions = []
|
| 442 |
+
for row in cursor.fetchall():
|
| 443 |
+
version = {
|
| 444 |
+
"id": row["id"],
|
| 445 |
+
"version_name": row["version_name"],
|
| 446 |
+
"base_model": row["base_model"],
|
| 447 |
+
"training_samples": row["training_samples"],
|
| 448 |
+
"created_at": row["created_at"],
|
| 449 |
+
"notes": row["notes"],
|
| 450 |
+
}
|
| 451 |
+
try:
|
| 452 |
+
version["metrics"] = json.loads(row["metrics_json"])
|
| 453 |
+
except (json.JSONDecodeError, TypeError):
|
| 454 |
+
version["metrics"] = {}
|
| 455 |
+
versions.append(version)
|
| 456 |
+
|
| 457 |
+
return versions
|
| 458 |
+
|
| 459 |
+
def get_latest_model(self) -> Optional[dict[str, Any]]:
|
| 460 |
+
"""الحصول على أحدث إصدار نموذج.
|
| 461 |
+
|
| 462 |
+
Returns:
|
| 463 |
+
قاموس الإصدار أو None إذا لم يوجد أي نموذج.
|
| 464 |
+
"""
|
| 465 |
+
versions = self.get_model_versions(limit=1)
|
| 466 |
+
return versions[0] if versions else None
|
| 467 |
+
|
| 468 |
+
# ------------------------------------------------------------------
|
| 469 |
+
# دورة الحياة
|
| 470 |
+
# ------------------------------------------------------------------
|
| 471 |
+
|
| 472 |
+
def close(self) -> None:
|
| 473 |
+
"""إغلاق اتصال قاعدة البيانات."""
|
| 474 |
+
if self._connection is not None:
|
| 475 |
+
self._connection.close()
|
| 476 |
+
self._connection = None
|
| 477 |
+
logger.debug("تم إغلاق اتصال قاعدة بيانات التعلم النشط")
|
| 478 |
+
|
| 479 |
+
def __enter__(self) -> "ActiveLearningDB":
|
| 480 |
+
"""Context manager entry."""
|
| 481 |
+
return self
|
| 482 |
+
|
| 483 |
+
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
| 484 |
+
"""Context manager exit."""
|
| 485 |
+
self.close()
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
# ======================================================================
|
| 489 |
+
# المتعلم النشط
|
| 490 |
+
# ======================================================================
|
| 491 |
+
|
| 492 |
+
class ActiveLearner:
|
| 493 |
+
"""نظام التعلم النشط لتحسين دقة OCR عبر الزمن.
|
| 494 |
+
|
| 495 |
+
يتتبع تصحيحات المستخدم، ويُصدر بيانات تدريب عند بلوغ عتبة
|
| 496 |
+
إعادة التدريب، ويسجّل إصدارات النماذج المُضبَّطة دقيقاً.
|
| 497 |
+
|
| 498 |
+
Attributes:
|
| 499 |
+
db: كائن ActiveLearningDB لإدارة التخزين.
|
| 500 |
+
retrain_threshold: عتبة عدد التصحيحات لإعادة التدريب.
|
| 501 |
+
"""
|
| 502 |
+
|
| 503 |
+
def __init__(
|
| 504 |
+
self,
|
| 505 |
+
db_path: str | Path = "data/active_learning.db",
|
| 506 |
+
retrain_threshold: int = 50,
|
| 507 |
+
) -> None:
|
| 508 |
+
"""تهيئة المتعلم النشط.
|
| 509 |
+
|
| 510 |
+
Args:
|
| 511 |
+
db_path: مسار قاعدة بيانات SQLite.
|
| 512 |
+
retrain_threshold: عدد التصحيحات المطلوب لتفعيل إعادة التدريب
|
| 513 |
+
(الافتراضي: 50).
|
| 514 |
+
"""
|
| 515 |
+
self.db = ActiveLearningDB(db_path)
|
| 516 |
+
self.retrain_threshold = retrain_threshold
|
| 517 |
+
logger.info(
|
| 518 |
+
"تم تهيئة المتعلم النشط (عتبة إعادة التدريب: %d)", retrain_threshold,
|
| 519 |
+
)
|
| 520 |
+
|
| 521 |
+
def log_correction(
|
| 522 |
+
self,
|
| 523 |
+
original_text: str,
|
| 524 |
+
corrected_text: str,
|
| 525 |
+
confidence: float = 0.0,
|
| 526 |
+
source: str = "",
|
| 527 |
+
image_data: Optional[bytes] = None,
|
| 528 |
+
) -> Optional[int]:
|
| 529 |
+
"""تسجيل تصحيح ��ن المستخدم مع إلغاء التكرار.
|
| 530 |
+
|
| 531 |
+
يُخزن التصحيح في قاعدة البيانات ويُنشئ سجل بيانات تدريب مرتبط.
|
| 532 |
+
إذا كان التصحيح مكرراً، يُتجاهل بصمت.
|
| 533 |
+
|
| 534 |
+
Args:
|
| 535 |
+
original_text: النص الأصلي من OCR.
|
| 536 |
+
corrected_text: التصحيح المُقدَّم من المستخدم.
|
| 537 |
+
confidence: مستوى ثقة OCR الأصلية (0.0 - 1.0).
|
| 538 |
+
source: محرك OCR المصدر (مثل "trocr", "easyocr").
|
| 539 |
+
image_data: بيانات الصورة المرتبطة (اختياري).
|
| 540 |
+
|
| 541 |
+
Returns:
|
| 542 |
+
معرّف التصحيح أو None إذا كان مكرراً.
|
| 543 |
+
|
| 544 |
+
Example:
|
| 545 |
+
>>> learner.log_correction("hellp", "hello", confidence=0.45, source="trocr")
|
| 546 |
+
"""
|
| 547 |
+
if not original_text.strip() or not corrected_text.strip():
|
| 548 |
+
logger.warning("تم تجاهل تصحيح فارغ")
|
| 549 |
+
return None
|
| 550 |
+
|
| 551 |
+
if original_text.strip().lower() == corrected_text.strip().lower():
|
| 552 |
+
logger.debug("النص الأصلي والمصحّح متطابقان — تم التجاهل")
|
| 553 |
+
return None
|
| 554 |
+
|
| 555 |
+
# إضافة إلى جدول التصحيحات
|
| 556 |
+
correction_id = self.db.add_correction(
|
| 557 |
+
original_text=original_text,
|
| 558 |
+
corrected_text=corrected_text,
|
| 559 |
+
original_confidence=confidence,
|
| 560 |
+
source_engine=source,
|
| 561 |
+
)
|
| 562 |
+
|
| 563 |
+
if correction_id is not None:
|
| 564 |
+
# إنشاء سجل بيانات تدريب مرتبط
|
| 565 |
+
self.db.add_training_entry(
|
| 566 |
+
correction_id=correction_id,
|
| 567 |
+
original_text=original_text.strip(),
|
| 568 |
+
corrected_text=corrected_text.strip(),
|
| 569 |
+
image_data=image_data,
|
| 570 |
+
source_engine=source,
|
| 571 |
+
)
|
| 572 |
+
|
| 573 |
+
# تسجيل تحذير عند اقتراب العتبة
|
| 574 |
+
total = self.db.count_corrections()
|
| 575 |
+
if total >= self.retrain_threshold:
|
| 576 |
+
logger.info(
|
| 577 |
+
"⚠️ بلغ عدد التصحيحات %d (العتبة: %d) — يُوصى بإعادة التدريب",
|
| 578 |
+
total, self.retrain_threshold,
|
| 579 |
+
)
|
| 580 |
+
|
| 581 |
+
return correction_id
|
| 582 |
+
|
| 583 |
+
def get_stats(self) -> dict[str, Any]:
|
| 584 |
+
"""الحصول على إحصائيات شاملة عن نظام التعلم النشط.
|
| 585 |
+
|
| 586 |
+
تتضمن:
|
| 587 |
+
- إجمالي التصحيحات
|
| 588 |
+
- متوسط الثقة قبل التصحيح
|
| 589 |
+
- عدد التصحيحات حسب المحرك
|
| 590 |
+
- حالة إعادة التدريب
|
| 591 |
+
- آخر إصدار نموذج
|
| 592 |
+
|
| 593 |
+
Returns:
|
| 594 |
+
قاموس يحتوي جميع الإحصائيات.
|
| 595 |
+
|
| 596 |
+
Example:
|
| 597 |
+
>>> stats = learner.get_stats()
|
| 598 |
+
>>> print(f"Corrections: {stats['total_corrections']}")
|
| 599 |
+
"""
|
| 600 |
+
corrections = self.db.get_corrections(limit=10000)
|
| 601 |
+
total = len(corrections)
|
| 602 |
+
|
| 603 |
+
if total == 0:
|
| 604 |
+
return {
|
| 605 |
+
"total_corrections": 0,
|
| 606 |
+
"should_retrain": False,
|
| 607 |
+
"retrain_threshold": self.retrain_threshold,
|
| 608 |
+
"avg_confidence_before": 0.0,
|
| 609 |
+
"corrections_by_engine": {},
|
| 610 |
+
"confidence_distribution": {},
|
| 611 |
+
"latest_model": None,
|
| 612 |
+
"model_versions_count": 0,
|
| 613 |
+
}
|
| 614 |
+
|
| 615 |
+
# حساب متوسط الثقة
|
| 616 |
+
confidences = [c["original_confidence"] for c in corrections if c["original_confidence"] > 0]
|
| 617 |
+
avg_confidence = sum(confidences) / len(confidences) if confidences else 0.0
|
| 618 |
+
|
| 619 |
+
# توزيع التصحيحات حسب المحرك
|
| 620 |
+
by_engine: dict[str, int] = {}
|
| 621 |
+
for c in corrections:
|
| 622 |
+
engine = c["source_engine"] or "unknown"
|
| 623 |
+
by_engine[engine] = by_engine.get(engine, 0) + 1
|
| 624 |
+
|
| 625 |
+
# توزيع الثقة
|
| 626 |
+
conf_dist: dict[str, int] = {"low": 0, "medium": 0, "high": 0}
|
| 627 |
+
for conf in confidences:
|
| 628 |
+
if conf < 0.5:
|
| 629 |
+
conf_dist["low"] += 1
|
| 630 |
+
elif conf < 0.8:
|
| 631 |
+
conf_dist["medium"] += 1
|
| 632 |
+
else:
|
| 633 |
+
conf_dist["high"] += 1
|
| 634 |
+
|
| 635 |
+
# آخر إصدار نموذج
|
| 636 |
+
latest_model = self.db.get_latest_model()
|
| 637 |
+
all_models = self.db.get_model_versions()
|
| 638 |
+
|
| 639 |
+
return {
|
| 640 |
+
"total_corrections": total,
|
| 641 |
+
"should_retrain": total >= self.retrain_threshold,
|
| 642 |
+
"retrain_threshold": self.retrain_threshold,
|
| 643 |
+
"retrain_progress": min(1.0, total / self.retrain_threshold) * 100,
|
| 644 |
+
"avg_confidence_before": round(avg_confidence, 4),
|
| 645 |
+
"corrections_by_engine": by_engine,
|
| 646 |
+
"confidence_distribution": conf_dist,
|
| 647 |
+
"latest_model": latest_model,
|
| 648 |
+
"model_versions_count": len(all_models),
|
| 649 |
+
"pending_training_samples": len(
|
| 650 |
+
self.db.get_training_data(limit=100000)
|
| 651 |
+
),
|
| 652 |
+
}
|
| 653 |
+
|
| 654 |
+
def should_retrain(self) -> bool:
|
| 655 |
+
"""فحص هل بلغ عدد التصحيحات عتبة إعادة التدريب.
|
| 656 |
+
|
| 657 |
+
Returns:
|
| 658 |
+
True إذا وصل العدد إلى العتبة أو تجاوزها.
|
| 659 |
+
"""
|
| 660 |
+
total = self.db.count_corrections()
|
| 661 |
+
result = total >= self.retrain_threshold
|
| 662 |
+
|
| 663 |
+
if result:
|
| 664 |
+
logger.info(
|
| 665 |
+
"✅ بلغ عدد التصحيحات %d/%d — جاهز لإعادة التدريب",
|
| 666 |
+
total, self.retrain_threshold,
|
| 667 |
+
)
|
| 668 |
+
|
| 669 |
+
return result
|
| 670 |
+
|
| 671 |
+
def export_training_data(
|
| 672 |
+
self,
|
| 673 |
+
output_path: str | Path,
|
| 674 |
+
include_image_data: bool = False,
|
| 675 |
+
min_confidence: float = 0.0,
|
| 676 |
+
) -> str:
|
| 677 |
+
"""تصدير بيانات التدريب بصيغة JSONL لضبط TrOCR الدقيق.
|
| 678 |
+
|
| 679 |
+
كل سطر في الملف يتضمن:
|
| 680 |
+
```json
|
| 681 |
+
{"original_text": "...", "corrected_text": "...", "source_engine": "..."}
|
| 682 |
+
```
|
| 683 |
+
|
| 684 |
+
Args:
|
| 685 |
+
output_path: مسار ملف JSONL المُخرَج.
|
| 686 |
+
include_image_data: تضمين بيانات الصور (base64) إن وُجدت.
|
| 687 |
+
min_confidence: أقل حد للثقة (لتصفية التصحيحات عالية الثقة).
|
| 688 |
+
|
| 689 |
+
Returns:
|
| 690 |
+
مسار الملف المُنشأ.
|
| 691 |
+
|
| 692 |
+
Raises:
|
| 693 |
+
ValueError: إذا لم توجد بيانات للتصدير.
|
| 694 |
+
|
| 695 |
+
Example:
|
| 696 |
+
>>> path = learner.export_training_data("training_data.jsonl")
|
| 697 |
+
>>> # استخدم الملف مع transformers Trainer:
|
| 698 |
+
>>> # trainer = Trainer(..., train_dataset=load_dataset("json", data_files=path))
|
| 699 |
+
"""
|
| 700 |
+
output_path = Path(output_path)
|
| 701 |
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 702 |
+
|
| 703 |
+
corrections = self.db.get_corrections(
|
| 704 |
+
min_confidence=min_confidence,
|
| 705 |
+
limit=100000,
|
| 706 |
+
)
|
| 707 |
+
|
| 708 |
+
if not corrections:
|
| 709 |
+
raise ValueError("لا توجد تصحيحات للتصدير")
|
| 710 |
+
|
| 711 |
+
lines_written = 0
|
| 712 |
+
with open(output_path, "w", encoding="utf-8") as f:
|
| 713 |
+
for correction in corrections:
|
| 714 |
+
entry = {
|
| 715 |
+
"original_text": correction["original_text"],
|
| 716 |
+
"corrected_text": correction["corrected_text"],
|
| 717 |
+
"source_engine": correction["source_engine"],
|
| 718 |
+
"original_confidence": correction["original_confidence"],
|
| 719 |
+
}
|
| 720 |
+
|
| 721 |
+
if include_image_data:
|
| 722 |
+
# محاولة استخراج بيانات الصورة من بيانات التدريب
|
| 723 |
+
training_entries = self.db.get_training_data(limit=100000)
|
| 724 |
+
for te in training_entries:
|
| 725 |
+
if te["correction_id"] == correction["id"]:
|
| 726 |
+
# ملاحظة: بيانات الصور تُخزن كـ BLOB ولا تُصدَّر في JSONL
|
| 727 |
+
# يمكن إضافتها كـ base64 إذا لزم الأمر
|
| 728 |
+
break
|
| 729 |
+
|
| 730 |
+
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
| 731 |
+
lines_written += 1
|
| 732 |
+
|
| 733 |
+
logger.info(
|
| 734 |
+
"تم تصدير %d سجل تدريب إلى: %s", lines_written, output_path,
|
| 735 |
+
)
|
| 736 |
+
return str(output_path)
|
| 737 |
+
|
| 738 |
+
def record_model_version(
|
| 739 |
+
self,
|
| 740 |
+
version_name: str,
|
| 741 |
+
metrics: Optional[dict[str, Any]] = None,
|
| 742 |
+
notes: str = "",
|
| 743 |
+
base_model: str = "",
|
| 744 |
+
) -> int:
|
| 745 |
+
"""تسجيل إصدار نموذج جديد بعد الضبط الدقيق.
|
| 746 |
+
|
| 747 |
+
يُحدَّث بيانات التدريب غير المستخدمة لتكون مرتبطة بهذا الإصدار.
|
| 748 |
+
|
| 749 |
+
Args:
|
| 750 |
+
version_name: اسم الإصدار (يجب أن يكون فريداً).
|
| 751 |
+
metrics: مقاييس الأداء (مثل {"cer": 0.03, "wer": 0.08}).
|
| 752 |
+
notes: ملاحظات إضافية.
|
| 753 |
+
base_model: اسم النموذج الأساسي قبل الضبط.
|
| 754 |
+
|
| 755 |
+
Returns:
|
| 756 |
+
معرّف الإصدار المسجّل.
|
| 757 |
+
|
| 758 |
+
Raises:
|
| 759 |
+
ValueError: إذا كان اسم الإصدار مكرراً.
|
| 760 |
+
|
| 761 |
+
Example:
|
| 762 |
+
>>> learner.record_model_version(
|
| 763 |
+
... "trocr-ar-v2",
|
| 764 |
+
... metrics={"cer": 0.032, "wer": 0.085, "accuracy": 0.968},
|
| 765 |
+
... notes="ضبط على 150 عيّنة عربية يدوية",
|
| 766 |
+
... )
|
| 767 |
+
"""
|
| 768 |
+
# تسجيل الإصدار
|
| 769 |
+
model_id = self.db.add_model_version(
|
| 770 |
+
version_name=version_name,
|
| 771 |
+
base_model=base_model,
|
| 772 |
+
training_samples=len(self.db.get_training_data(limit=100000)),
|
| 773 |
+
metrics=metrics,
|
| 774 |
+
notes=notes,
|
| 775 |
+
)
|
| 776 |
+
|
| 777 |
+
# تحديد بيانات التدريب المُستخدمة
|
| 778 |
+
self.db.mark_training_used(version_name)
|
| 779 |
+
|
| 780 |
+
logger.info(
|
| 781 |
+
"✅ تم تسجيل إصدار النموذج '%s' (ID: %d) مع %d عيّنة تدريب",
|
| 782 |
+
version_name, model_id,
|
| 783 |
+
len(self.db.get_model_versions(limit=1)),
|
| 784 |
+
)
|
| 785 |
+
|
| 786 |
+
return model_id
|
| 787 |
+
|
| 788 |
+
def get_confidence_improvement(self) -> Optional[dict[str, float]]:
|
| 789 |
+
"""حساب تحسّن الثقة بين إصدارات النماذج.
|
| 790 |
+
|
| 791 |
+
يُقارن مقاييس CER/WER بين آخر إصدارين.
|
| 792 |
+
|
| 793 |
+
Returns:
|
| 794 |
+
قاموس التحسّن أو None إذا لم يوجد إصداران للمقارنة.
|
| 795 |
+
|
| 796 |
+
Example:
|
| 797 |
+
>>> improvement = learner.get_confidence_improvement()
|
| 798 |
+
>>> if improvement:
|
| 799 |
+
... print(f"CER improved by: {improvement['cer_delta']:.2%}")
|
| 800 |
+
"""
|
| 801 |
+
versions = self.db.get_model_versions(limit=2)
|
| 802 |
+
if len(versions) < 2:
|
| 803 |
+
logger.debug("لا يوجد إصداران كافيان لمقارنة التحسّن")
|
| 804 |
+
return None
|
| 805 |
+
|
| 806 |
+
latest = versions[0]
|
| 807 |
+
previous = versions[1]
|
| 808 |
+
|
| 809 |
+
improvement: dict[str, float] = {}
|
| 810 |
+
for metric_key in set(list(latest["metrics"].keys()) + list(previous["metrics"].keys())):
|
| 811 |
+
latest_val = latest["metrics"].get(metric_key)
|
| 812 |
+
prev_val = previous["metrics"].get(metric_key)
|
| 813 |
+
|
| 814 |
+
if latest_val is not None and prev_val is not None:
|
| 815 |
+
try:
|
| 816 |
+
delta = float(prev_val) - float(latest_val)
|
| 817 |
+
improvement[f"{metric_key}_delta"] = delta
|
| 818 |
+
improvement[f"{metric_key}_delta_pct"] = (
|
| 819 |
+
delta / float(prev_val) * 100 if float(prev_val) != 0 else 0.0
|
| 820 |
+
)
|
| 821 |
+
except (ValueError, ZeroDivisionError):
|
| 822 |
+
continue
|
| 823 |
+
|
| 824 |
+
if improvement:
|
| 825 |
+
logger.info("تحسّن الأداء بين الإصدارات: %s", improvement)
|
| 826 |
+
return improvement
|
| 827 |
+
|
| 828 |
+
return None
|
| 829 |
+
|
| 830 |
+
def close(self) -> None:
|
| 831 |
+
"""إغلاق قاعدة البيانات وتحرير الموارد."""
|
| 832 |
+
self.db.close()
|
| 833 |
+
|
| 834 |
+
def __enter__(self) -> "ActiveLearner":
|
| 835 |
+
"""Context manager entry."""
|
| 836 |
+
return self
|
| 837 |
+
|
| 838 |
+
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
| 839 |
+
"""Context manager exit."""
|
| 840 |
+
self.close()
|
|
@@ -11,6 +11,8 @@
|
|
| 11 |
- معالجة النصوص العربية من اليمين لليسار (RTL)
|
| 12 |
- معالجة النصوص المختلطة (عربي + إنجليزي + أرقام)
|
| 13 |
- تصحيح النصوص باستخدام الذكاء الاصطناعي (GPT)
|
|
|
|
|
|
|
| 14 |
"""
|
| 15 |
from modules.nlp.text_classifier import TextClassifier
|
| 16 |
from modules.nlp.entity_extractor import EntityExtractor
|
|
@@ -23,6 +25,8 @@ from modules.nlp.mixed_text import (
|
|
| 23 |
optimize_mixed_text,
|
| 24 |
separate_text_components,
|
| 25 |
)
|
|
|
|
|
|
|
| 26 |
try:
|
| 27 |
from modules.nlp.ai_corrector import AICorrector
|
| 28 |
except ImportError: # تبعيات اختيارية مثل python-dotenv / openai
|
|
@@ -33,6 +37,8 @@ __all__ = [
|
|
| 33 |
"SpellCorrector", "LanguageDetector",
|
| 34 |
"RTLFixer", "is_rtl_text", "get_text_direction",
|
| 35 |
"detect_language", "optimize_mixed_text", "separate_text_components",
|
|
|
|
|
|
|
| 36 |
]
|
| 37 |
|
| 38 |
if AICorrector is not None:
|
|
|
|
| 11 |
- معالجة النصوص العربية من اليمين لليسار (RTL)
|
| 12 |
- معالجة النصوص المختلطة (عربي + إنجليزي + أرقام)
|
| 13 |
- تصحيح النصوص باستخدام الذكاء الاصطناعي (GPT)
|
| 14 |
+
- نظام الكلمات المحمية الشامل (Protected Words)
|
| 15 |
+
- مولّد المراجع الدراسية (Study Guide Generator)
|
| 16 |
"""
|
| 17 |
from modules.nlp.text_classifier import TextClassifier
|
| 18 |
from modules.nlp.entity_extractor import EntityExtractor
|
|
|
|
| 25 |
optimize_mixed_text,
|
| 26 |
separate_text_components,
|
| 27 |
)
|
| 28 |
+
from modules.nlp.protected_words import ProtectedWordsManager
|
| 29 |
+
from modules.nlp.study_guide import StudyGuideGenerator
|
| 30 |
try:
|
| 31 |
from modules.nlp.ai_corrector import AICorrector
|
| 32 |
except ImportError: # تبعيات اختيارية مثل python-dotenv / openai
|
|
|
|
| 37 |
"SpellCorrector", "LanguageDetector",
|
| 38 |
"RTLFixer", "is_rtl_text", "get_text_direction",
|
| 39 |
"detect_language", "optimize_mixed_text", "separate_text_components",
|
| 40 |
+
"ProtectedWordsManager",
|
| 41 |
+
"StudyGuideGenerator",
|
| 42 |
]
|
| 43 |
|
| 44 |
if AICorrector is not None:
|
|
@@ -0,0 +1,653 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
نظام الكلمات المحمية — Protected Words Module
|
| 3 |
+
==================================================
|
| 4 |
+
حماية المصطلحات المتخصصة من التصحيح الإملائي التلقائي.
|
| 5 |
+
|
| 6 |
+
القدرات:
|
| 7 |
+
- مجموعات محددة مسبقاً: تقنية (~60)، بايثون (~60)، طبية (~40)
|
| 8 |
+
- دعم مصطلحات مخصصة: إضافة/إزالة/فحص
|
| 9 |
+
- تكامل مع المصحح الإملائي: فلترة الكلمات المحمية
|
| 10 |
+
- فئات شاملة: برمجة، طبية، علمية، أسماء، اختصارات
|
| 11 |
+
- حفظ/تحميل المصطلحات المخصصة بصيغة JSON
|
| 12 |
+
- البحث بأحرف مطابقة (case-insensitive)
|
| 13 |
+
|
| 14 |
+
مثال الاستخدام:
|
| 15 |
+
>>> manager = ProtectedWordsManager()
|
| 16 |
+
>>> manager.add_words(["React", "Kubernetes", "TrOCR"], category="custom")
|
| 17 |
+
>>> if manager.is_protected("React"):
|
| 18 |
+
... print("محمي!")
|
| 19 |
+
>>> to_skip = manager.filter_protected(["hello", "React", "numpy", "word"])
|
| 20 |
+
>>> # to_skip == ["React", "numpy"]
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import json
|
| 26 |
+
import logging
|
| 27 |
+
import os
|
| 28 |
+
from collections import defaultdict
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
from typing import Any, Optional
|
| 31 |
+
|
| 32 |
+
logger = logging.getLogger(__name__)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# ======================================================================
|
| 36 |
+
# مجموعات الكلمات المحددة مسبقاً
|
| 37 |
+
# ======================================================================
|
| 38 |
+
|
| 39 |
+
TECHNICAL_KEYWORDS: set[str] = {
|
| 40 |
+
# مصطلحات برمجية عامة
|
| 41 |
+
"python", "pythonistas", "scraping", "parsing", "ocr",
|
| 42 |
+
"batch", "programming", "script", "database", "configure",
|
| 43 |
+
"setup", "env", "immutable", "concatenation", "tuples",
|
| 44 |
+
"dictionaries", "debugging", "programmatically", "spreadsheet",
|
| 45 |
+
"integers", "boolean", "syntax", "web",
|
| 46 |
+
"etl", "dataframe", "json", "csv", "yaml", "markdown",
|
| 47 |
+
"mermaid", "repository", "clone", "commit", "push",
|
| 48 |
+
# اختصارات تقنية
|
| 49 |
+
"repl", "dpi", "api", "gpu", "cpu", "ram", "rom",
|
| 50 |
+
"lora", "huggingface", "transformers", "pytorch", "tensorboard",
|
| 51 |
+
# مفاهيم تقنية
|
| 52 |
+
"comprehensions", "replication", "precedence", "modulo",
|
| 53 |
+
"exponent", "traceback", "overriding",
|
| 54 |
+
# مكتبات بايثون شائعة
|
| 55 |
+
"numpy", "pandas", "matplotlib", "scipy", "sklearn",
|
| 56 |
+
"opencv", "pillow", "tqdm", "requests", "beautifulsoup",
|
| 57 |
+
# DevOps و بنية
|
| 58 |
+
"docker", "kubernetes", "git", "npm", "pip", "conda",
|
| 59 |
+
"ci", "cd", "devops", "ansible", "terraform", "jenkins",
|
| 60 |
+
# أطر عمل
|
| 61 |
+
"flask", "django", "fastapi", "uvicorn", "gradio",
|
| 62 |
+
"react", "vue", "angular", "nextjs", "express",
|
| 63 |
+
# قواعد بيانات
|
| 64 |
+
"mongodb", "postgresql", "mysql", "sqlite", "redis",
|
| 65 |
+
"sqlalchemy", "pymysql", "psycopg2",
|
| 66 |
+
# تعلم آلي
|
| 67 |
+
"machinelearning", "deeplearning", "neuralnetwork",
|
| 68 |
+
"tensorflow", "keras", "pytorch", "sklearn", "xgboost",
|
| 69 |
+
"randomforest", "gradientboost", "sigmoid", "relu",
|
| 70 |
+
"softmax", "backpropagation", "overfitting", "underfitting",
|
| 71 |
+
"huggingface", "transformers", "datasets", "tokenizers",
|
| 72 |
+
# سحابة
|
| 73 |
+
"aws", "gcp", "azure", "heroku", "vercel", "netlify",
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
PYTHON_KEYWORDS: set[str] = {
|
| 77 |
+
# كلمات محجوزة رسمياً
|
| 78 |
+
"False", "None", "True", "and", "as", "assert", "async", "await",
|
| 79 |
+
"break", "class", "continue", "def", "del", "elif", "else", "except",
|
| 80 |
+
"finally", "for", "from", "global", "if", "import", "in", "is",
|
| 81 |
+
"lambda", "nonlocal", "not", "or", "pass", "raise", "return",
|
| 82 |
+
"try", "while", "with", "yield",
|
| 83 |
+
# دوال مدمجة (builtins)
|
| 84 |
+
"print", "input", "len", "range", "type", "int", "str", "float",
|
| 85 |
+
"list", "dict", "set", "tuple", "bool", "open", "file", "super",
|
| 86 |
+
"self", "cls", "repr", "main", "name", "args", "kwargs",
|
| 87 |
+
"append", "extend", "pop", "sort", "join", "split", "strip",
|
| 88 |
+
"format", "replace", "lower", "upper", "title", "capitalize",
|
| 89 |
+
"enumerate", "zip", "map", "filter", "sorted", "reversed",
|
| 90 |
+
"isinstance", "issubclass", "hasattr", "getattr", "setattr",
|
| 91 |
+
"module", "package", "staticmethod", "classmethod", "property",
|
| 92 |
+
"abs", "all", "any", "bin", "chr", "dir", "eval", "exec",
|
| 93 |
+
"getattr", "hex", "id", "max", "min", "oct", "ord", "pow",
|
| 94 |
+
"repr", "round", "sum", "vars", "iter", "next",
|
| 95 |
+
# استثناءات
|
| 96 |
+
"Exception", "ValueError", "TypeError", "KeyError", "IndexError",
|
| 97 |
+
"AttributeError", "RuntimeError", "StopIteration",
|
| 98 |
+
"NotImplementedError", "IOError", "OSError", "ImportError",
|
| 99 |
+
"ModuleNotFoundError", "FileNotFoundError", "PermissionError",
|
| 100 |
+
"ZeroDivisionError", "NameError", "UnicodeDecodeError",
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
MEDICAL_TERMS: set[str] = {
|
| 104 |
+
# تشريح
|
| 105 |
+
"cardiology", "neurology", "dermatology", "ophthalmology",
|
| 106 |
+
"orthopedics", "pulmonology", "gastroenterology", "endocrinology",
|
| 107 |
+
"hematology", "oncology", "nephrology", "rheumatology",
|
| 108 |
+
"pathology", "radiology", "anesthesiology", "psychiatry",
|
| 109 |
+
# مصطل��ات طبية
|
| 110 |
+
"hypertension", "diabetes", "cholesterol", "antibiotic",
|
| 111 |
+
"inflammation", "metabolism", "symptom", "diagnosis",
|
| 112 |
+
"prognosis", "biopsy", "palliative", "prophylactic",
|
| 113 |
+
"benign", "malignant", "chronic", "acute", "subacute",
|
| 114 |
+
"congenital", "hereditary", "idiopathic", "iatrogenic",
|
| 115 |
+
# أدوية
|
| 116 |
+
"ibuprofen", "acetaminophen", "amoxicillin", "metformin",
|
| 117 |
+
"atorvastatin", "omeprazole", "lisinopril", "aspirin",
|
| 118 |
+
"prednisone", "morphine", "insulin", "penicillin",
|
| 119 |
+
# إجراءات
|
| 120 |
+
"laparoscopy", "endoscopy", "biopsy", "transplant",
|
| 121 |
+
"angioplasty", "arthroscopy", "colonoscopy", "mammography",
|
| 122 |
+
# مختبر
|
| 123 |
+
"hemoglobin", "leukocyte", "erythrocyte", "thrombocyte",
|
| 124 |
+
"electrolyte", "bilirubin", "creatinine", "glucose",
|
| 125 |
+
"potassium", "sodium", "calcium", "phosphate",
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
SCIENTIFIC_TERMS: set[str] = {
|
| 129 |
+
# فيزياء
|
| 130 |
+
"quantum", "relativity", "entropy", "enthalpy",
|
| 131 |
+
"wavelength", "frequency", "amplitude", "velocity",
|
| 132 |
+
"acceleration", "momentum", "thermodynamics", "electromagnetic",
|
| 133 |
+
# كيمياء
|
| 134 |
+
"catalyst", "polymer", "isotope", "electrolyte",
|
| 135 |
+
"oxidation", "reduction", "synthesis", "titration",
|
| 136 |
+
"covalent", "ionic", "molecular", "stoichiometry",
|
| 137 |
+
# أحياء
|
| 138 |
+
"mitosis", "meiosis", "photosynthesis", "respiration",
|
| 139 |
+
"chromosome", "genotype", "phenotype", "mutation",
|
| 140 |
+
"biodiversity", "ecosystem", "symbiosis", "homeostasis",
|
| 141 |
+
# رياضيات
|
| 142 |
+
"algorithm", "polynomial", "derivative", "integral",
|
| 143 |
+
"correlation", "regression", "variance", "deviation",
|
| 144 |
+
"hypothesis", "probability", "permutation", "combination",
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
ABBREVIATIONS: set[str] = {
|
| 148 |
+
# تقنية
|
| 149 |
+
"HTML", "CSS", "JS", "TS", "XML", "HTTP", "HTTPS", "FTP",
|
| 150 |
+
"SSH", "SSL", "TLS", "TCP", "UDP", "DNS", "DHCP",
|
| 151 |
+
"REST", "SOAP", "GraphQL", "JSON", "YAML", "TOML",
|
| 152 |
+
"IDE", "CLI", "GUI", "API", "SDK", "SDKs",
|
| 153 |
+
"SQL", "NoSQL", "ORM", "ODM",
|
| 154 |
+
# عامة
|
| 155 |
+
"NASA", "IEEE", "ISO", "UTF", "ASCII", "Unicode",
|
| 156 |
+
"URL", "URI", "URN", "PDF", "CSV", "JSONL",
|
| 157 |
+
"GPU", "TPU", "NPU", "SSD", "HDD", "RAM", "ROM",
|
| 158 |
+
"LAN", "WAN", "VPN", "IoT", "SaaS", "PaaS", "IaaS",
|
| 159 |
+
# أوائل كلمات طبية وعلمية
|
| 160 |
+
"CT", "MRI", "PET", "ECG", "EEG", "EMG",
|
| 161 |
+
"DNA", "RNA", "mRNA", "tRNA", "rRNA",
|
| 162 |
+
"WHO", "CDC", "FDA", "NIH", "AMA",
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
# ======================================================================
|
| 167 |
+
# فئة مدير الكلمات المحمية
|
| 168 |
+
# ======================================================================
|
| 169 |
+
|
| 170 |
+
class ProtectedWordsManager:
|
| 171 |
+
"""مدير شامل للكلمات المحمية من التصحيح الإملائي.
|
| 172 |
+
|
| 173 |
+
يوفر واجهة موحدة لإدارة المصطلحات المتخصصة وتكاملها
|
| 174 |
+
مع المصحح الإملائي.
|
| 175 |
+
|
| 176 |
+
Attributes:
|
| 177 |
+
categories: قاموس الفئات ومصطلحاتها.
|
| 178 |
+
custom_vocabulary: المصطلحات المخصصة.
|
| 179 |
+
"""
|
| 180 |
+
|
| 181 |
+
# الفئات المدعومة مع مجموعاتها الافتراضية
|
| 182 |
+
DEFAULT_CATEGORIES: dict[str, set[str]] = {
|
| 183 |
+
"programming": TECHNICAL_KEYWORDS,
|
| 184 |
+
"python": PYTHON_KEYWORDS,
|
| 185 |
+
"medical": MEDICAL_TERMS,
|
| 186 |
+
"scientific": SCIENTIFIC_TERMS,
|
| 187 |
+
"abbreviations": ABBREVIATIONS,
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
def __init__(
|
| 191 |
+
self,
|
| 192 |
+
custom_vocabulary_path: Optional[str | Path] = None,
|
| 193 |
+
additional_categories: Optional[dict[str, set[str]]] = None,
|
| 194 |
+
) -> None:
|
| 195 |
+
"""تهيئة مدير الكلمات المحمية.
|
| 196 |
+
|
| 197 |
+
Args:
|
| 198 |
+
custom_vocabulary_path: مسار ملف المصطلحات المخصصة (JSON).
|
| 199 |
+
additional_categories: فئات إضافية مخصصة.
|
| 200 |
+
"""
|
| 201 |
+
# نسخ المجموعات الافتراضية
|
| 202 |
+
self.categories: dict[str, set[str]] = {
|
| 203 |
+
name: set(words) for name, words in self.DEFAULT_CATEGORIES.items()
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
# إضافة فئات مخصصة
|
| 207 |
+
if additional_categories:
|
| 208 |
+
for cat_name, words in additional_categories.items():
|
| 209 |
+
if cat_name in self.categories:
|
| 210 |
+
self.categories[cat_name].update(words)
|
| 211 |
+
else:
|
| 212 |
+
self.categories[cat_name] = set(words)
|
| 213 |
+
|
| 214 |
+
# المصطلحات المخصصة
|
| 215 |
+
self.custom_vocabulary: dict[str, str] = {} # word -> category
|
| 216 |
+
|
| 217 |
+
# تحميل المصطلحات المخصصة من ملف
|
| 218 |
+
if custom_vocabulary_path:
|
| 219 |
+
self.load_custom_vocabulary(custom_vocabulary_path)
|
| 220 |
+
|
| 221 |
+
# إعادة بناء الفهرس
|
| 222 |
+
self._protected_index: set[str] = set()
|
| 223 |
+
self._rebuild_index()
|
| 224 |
+
|
| 225 |
+
logger.info(
|
| 226 |
+
"تم تهيئة مدير الكلمات المحمية: %d فئة, %d كلمة إجمالي",
|
| 227 |
+
len(self.categories), len(self._protected_index),
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
def _rebuild_index(self) -> None:
|
| 231 |
+
"""إعادة بناء الفهرس الموحد للبحث السريع (case-insensitive)."""
|
| 232 |
+
self._protected_index = set()
|
| 233 |
+
|
| 234 |
+
for words in self.categories.values():
|
| 235 |
+
self._protected_index.update(w.lower() for w in words)
|
| 236 |
+
|
| 237 |
+
self._protected_index.update(w.lower() for w in self.custom_vocabulary)
|
| 238 |
+
|
| 239 |
+
# ------------------------------------------------------------------
|
| 240 |
+
# فحص الحماية
|
| 241 |
+
# ------------------------------------------------------------------
|
| 242 |
+
|
| 243 |
+
def is_protected(self, word: str) -> bool:
|
| 244 |
+
"""فحص هل الكلمة محمية من التصحيح.
|
| 245 |
+
|
| 246 |
+
البحث بأحرف مطابقة (case-insensitive).
|
| 247 |
+
|
| 248 |
+
Args:
|
| 249 |
+
word: الكلمة المراد فحصها.
|
| 250 |
+
|
| 251 |
+
Returns:
|
| 252 |
+
True إذا كانت محمية.
|
| 253 |
+
|
| 254 |
+
Example:
|
| 255 |
+
>>> manager.is_protected("React") # True
|
| 256 |
+
>>> manager.is_protected("hello") # False
|
| 257 |
+
"""
|
| 258 |
+
return word.lower() in self._protected_index
|
| 259 |
+
|
| 260 |
+
def get_category(self, word: str) -> Optional[str]:
|
| 261 |
+
"""الحصول على فئة الكلمة المحمية.
|
| 262 |
+
|
| 263 |
+
Args:
|
| 264 |
+
word: الكلمة المراد فحصها.
|
| 265 |
+
|
| 266 |
+
Returns:
|
| 267 |
+
اسم الفئة أو None إذا لم تكن محمية.
|
| 268 |
+
"""
|
| 269 |
+
word_lower = word.lower()
|
| 270 |
+
|
| 271 |
+
# فحص الفئات المحددة
|
| 272 |
+
for cat_name, words in self.categories.items():
|
| 273 |
+
if word_lower in {w.lower() for w in words}:
|
| 274 |
+
return cat_name
|
| 275 |
+
|
| 276 |
+
# فحص المصطلحات المخصصة
|
| 277 |
+
if word_lower in {w.lower() for w in self.custom_vocabulary}:
|
| 278 |
+
return self.custom_vocabulary.get(word, "custom")
|
| 279 |
+
|
| 280 |
+
return None
|
| 281 |
+
|
| 282 |
+
# ------------------------------------------------------------------
|
| 283 |
+
# إدارة المصطلحات المخصصة
|
| 284 |
+
# ------------------------------------------------------------------
|
| 285 |
+
|
| 286 |
+
def add_words(
|
| 287 |
+
self,
|
| 288 |
+
words: list[str],
|
| 289 |
+
category: str = "custom",
|
| 290 |
+
) -> int:
|
| 291 |
+
"""إضافة كلمات إلى القائمة المحمية.
|
| 292 |
+
|
| 293 |
+
Args:
|
| 294 |
+
words: قائمة الكلمات المطلوب حمايتها.
|
| 295 |
+
category: الفئة (مثل "custom", "names", "domain_specific").
|
| 296 |
+
|
| 297 |
+
Returns:
|
| 298 |
+
عدد الكلمات المُضافة فعلياً (تجاهل التكرار).
|
| 299 |
+
|
| 300 |
+
Example:
|
| 301 |
+
>>> manager.add_words(["TrOCR", "Surya", "Kitab"], category="ocr")
|
| 302 |
+
"""
|
| 303 |
+
added = 0
|
| 304 |
+
for word in words:
|
| 305 |
+
word = word.strip()
|
| 306 |
+
if not word:
|
| 307 |
+
continue
|
| 308 |
+
|
| 309 |
+
word_lower = word.lower()
|
| 310 |
+
if word_lower in self._protected_index:
|
| 311 |
+
logger.debug("كلمة محمية بالفعل: '%s'", word)
|
| 312 |
+
continue
|
| 313 |
+
|
| 314 |
+
self.custom_vocabulary[word] = category
|
| 315 |
+
self._protected_index.add(word_lower)
|
| 316 |
+
added += 1
|
| 317 |
+
|
| 318 |
+
# إضافة الفئة إذا لم تكن موجودة
|
| 319 |
+
if category != "custom" and category not in self.categories:
|
| 320 |
+
self.categories[category] = set()
|
| 321 |
+
|
| 322 |
+
if added > 0:
|
| 323 |
+
logger.info(
|
| 324 |
+
"تمت إضافة %d كلمة محمية (الفئة: '%s')",
|
| 325 |
+
added, category,
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
return added
|
| 329 |
+
|
| 330 |
+
def remove_words(self, words: list[str]) -> int:
|
| 331 |
+
"""إزالة كلمات من القائمة المحمية (المخصصة فقط).
|
| 332 |
+
|
| 333 |
+
لا يمكن إزالة كلمات من المجموعات الافتراضية المحددة مسبقاً.
|
| 334 |
+
|
| 335 |
+
Args:
|
| 336 |
+
words: قائمة الكلمات المراد إزالتها.
|
| 337 |
+
|
| 338 |
+
Returns:
|
| 339 |
+
عدد الكلمات المُزالة فعلياً.
|
| 340 |
+
|
| 341 |
+
Example:
|
| 342 |
+
>>> manager.remove_words(["old_term", "deprecated_word"])
|
| 343 |
+
"""
|
| 344 |
+
removed = 0
|
| 345 |
+
|
| 346 |
+
for word in words:
|
| 347 |
+
word = word.strip()
|
| 348 |
+
if not word:
|
| 349 |
+
continue
|
| 350 |
+
|
| 351 |
+
# إزالة من المصطلحات المخصصة فقط
|
| 352 |
+
if word in self.custom_vocabulary:
|
| 353 |
+
del self.custom_vocabulary[word]
|
| 354 |
+
self._protected_index.discard(word.lower())
|
| 355 |
+
removed += 1
|
| 356 |
+
logger.debug("تمت إزالة: '%s'", word)
|
| 357 |
+
else:
|
| 358 |
+
# فحص هل الكلمة من المجموعات الافتراضية
|
| 359 |
+
in_default = False
|
| 360 |
+
for cat_words in self.DEFAULT_CATEGORIES.values():
|
| 361 |
+
if word.lower() in {w.lower() for w in cat_words}:
|
| 362 |
+
in_default = True
|
| 363 |
+
break
|
| 364 |
+
|
| 365 |
+
if in_default:
|
| 366 |
+
logger.debug(
|
| 367 |
+
"لا يمكن إزالة '%s' — من مجموعة محددة مسبقاً",
|
| 368 |
+
word,
|
| 369 |
+
)
|
| 370 |
+
else:
|
| 371 |
+
# ليست محمية أصلاً
|
| 372 |
+
pass
|
| 373 |
+
|
| 374 |
+
if removed > 0:
|
| 375 |
+
self._rebuild_index()
|
| 376 |
+
|
| 377 |
+
return removed
|
| 378 |
+
|
| 379 |
+
# ------------------------------------------------------------------
|
| 380 |
+
# تكامل مع المصحح الإملائي
|
| 381 |
+
# ------------------------------------------------------------------
|
| 382 |
+
|
| 383 |
+
def filter_protected(
|
| 384 |
+
self,
|
| 385 |
+
words: list[str],
|
| 386 |
+
include_category: bool = False,
|
| 387 |
+
) -> list[str | dict[str, str]]:
|
| 388 |
+
"""فلترة الكلمات المحمية من قائمة كلمات.
|
| 389 |
+
|
| 390 |
+
يُستخدم لتحديد الكلمات التي يجب تخطيها أثناء التصحيح الإملائي.
|
| 391 |
+
|
| 392 |
+
Args:
|
| 393 |
+
words: قائمة الكلمات المراد فحصها.
|
| 394 |
+
include_category: تضمين الفئة في النتيجة.
|
| 395 |
+
|
| 396 |
+
Returns:
|
| 397 |
+
قائمة الكلمات المحمية (أو قواميس مع الفئة).
|
| 398 |
+
|
| 399 |
+
Example:
|
| 400 |
+
>>> to_skip = manager.filter_protected(["hello", "numpy", "world"])
|
| 401 |
+
>>> # to_skip == ["numpy"]
|
| 402 |
+
>>> detailed = manager.filter_protected(["hello", "numpy"], include_category=True)
|
| 403 |
+
>>> # detailed == [{"word": "numpy", "category": "programming"}]
|
| 404 |
+
"""
|
| 405 |
+
protected = []
|
| 406 |
+
|
| 407 |
+
for word in words:
|
| 408 |
+
if self.is_protected(word):
|
| 409 |
+
if include_category:
|
| 410 |
+
cat = self.get_category(word) or "unknown"
|
| 411 |
+
protected.append({"word": word, "category": cat})
|
| 412 |
+
else:
|
| 413 |
+
protected.append(word)
|
| 414 |
+
|
| 415 |
+
return protected
|
| 416 |
+
|
| 417 |
+
def get_safe_words(
|
| 418 |
+
self,
|
| 419 |
+
words: list[str],
|
| 420 |
+
) -> list[str]:
|
| 421 |
+
"""الحصول على الكلمات غير المحمية (الآمنة للتصحيح).
|
| 422 |
+
|
| 423 |
+
Args:
|
| 424 |
+
words: قائمة الكلمات.
|
| 425 |
+
|
| 426 |
+
Returns:
|
| 427 |
+
قائمة الكلمات غير المحمية.
|
| 428 |
+
|
| 429 |
+
Example:
|
| 430 |
+
>>> safe = manager.get_safe_words(["hello", "numpy", "world"])
|
| 431 |
+
>>> # safe == ["hello", "world"]
|
| 432 |
+
"""
|
| 433 |
+
return [w for w in words if not self.is_protected(w)]
|
| 434 |
+
|
| 435 |
+
def protect_text(self, text: str, placeholder: str = "█") -> tuple[str, dict[str, str]]:
|
| 436 |
+
"""حماية المصطلحات في نص عن طريق استبدالها بعناصر نائبة.
|
| 437 |
+
|
| 438 |
+
مفيد لتصحيح نص دون المساس بالمصطلحات.
|
| 439 |
+
|
| 440 |
+
Args:
|
| 441 |
+
text: النص الأصلي.
|
| 442 |
+
placeholder: العنصر النائب (الافتراضي: █).
|
| 443 |
+
|
| 444 |
+
Returns:
|
| 445 |
+
tuple (النص المحمي، قاموس {عنصر_نائب: كلمة_أصلية}).
|
| 446 |
+
|
| 447 |
+
Example:
|
| 448 |
+
>>> protected_text, mapping = manager.protect_text("using numpy and pandas")
|
| 449 |
+
>>> # protected_text ≈ "using █ and █"
|
| 450 |
+
>>> # mapping = {"█_0": "numpy", "█_1": "pandas"}
|
| 451 |
+
"""
|
| 452 |
+
import re
|
| 453 |
+
|
| 454 |
+
# تجزئة النص إلى كلمات مع الفواصل
|
| 455 |
+
tokens: list[tuple[str, str]] = []
|
| 456 |
+
pattern = re.compile(r"(\S+)(\s*)")
|
| 457 |
+
for match in pattern.finditer(text):
|
| 458 |
+
tokens.append((match.group(1), match.group(2)))
|
| 459 |
+
|
| 460 |
+
mapping: dict[str, str] = {}
|
| 461 |
+
result_tokens: list[str] = []
|
| 462 |
+
placeholder_counter = 0
|
| 463 |
+
|
| 464 |
+
for word, separator in tokens:
|
| 465 |
+
if self.is_protected(word):
|
| 466 |
+
placeholder_key = f"{placeholder}_{placeholder_counter}"
|
| 467 |
+
mapping[placeholder_key] = word
|
| 468 |
+
result_tokens.append(placeholder_key)
|
| 469 |
+
placeholder_counter += 1
|
| 470 |
+
else:
|
| 471 |
+
result_tokens.append(word)
|
| 472 |
+
|
| 473 |
+
result_tokens.append(separator)
|
| 474 |
+
|
| 475 |
+
protected_text = "".join(result_tokens)
|
| 476 |
+
return protected_text, mapping
|
| 477 |
+
|
| 478 |
+
def restore_text(self, text: str, mapping: dict[str, str]) -> str:
|
| 479 |
+
"""استعادة الكلمات المحمية في نص بعد التصحيح.
|
| 480 |
+
|
| 481 |
+
Args:
|
| 482 |
+
text: النص بعد التصحيح (مع العناصر النائبة).
|
| 483 |
+
mapping: القاموس من protect_text().
|
| 484 |
+
|
| 485 |
+
Returns:
|
| 486 |
+
النص المُستعاد.
|
| 487 |
+
"""
|
| 488 |
+
result = text
|
| 489 |
+
for placeholder, original in mapping.items():
|
| 490 |
+
result = result.replace(placeholder, original)
|
| 491 |
+
return result
|
| 492 |
+
|
| 493 |
+
# ------------------------------------------------------------------
|
| 494 |
+
# إحصائيات ومعلومات
|
| 495 |
+
# ------------------------------------------------------------------
|
| 496 |
+
|
| 497 |
+
def get_stats(self) -> dict[str, Any]:
|
| 498 |
+
"""الحصول على إحصائيات عن الكلمات المحمية.
|
| 499 |
+
|
| 500 |
+
Returns:
|
| 501 |
+
قاموس يحتوي:
|
| 502 |
+
- total_protected: العدد الإجمالي
|
| 503 |
+
- by_category: التوزيع حسب الفئة
|
| 504 |
+
- custom_count: عدد المصطلحات المخصصة
|
| 505 |
+
"""
|
| 506 |
+
by_category: dict[str, int] = {}
|
| 507 |
+
for cat_name, words in self.categories.items():
|
| 508 |
+
by_category[cat_name] = len(words)
|
| 509 |
+
|
| 510 |
+
if self.custom_vocabulary:
|
| 511 |
+
by_category["custom_user"] = len(self.custom_vocabulary)
|
| 512 |
+
|
| 513 |
+
return {
|
| 514 |
+
"total_protected": len(self._protected_index),
|
| 515 |
+
"by_category": by_category,
|
| 516 |
+
"custom_count": len(self.custom_vocabulary),
|
| 517 |
+
}
|
| 518 |
+
|
| 519 |
+
def get_words_by_category(self, category: str) -> list[str]:
|
| 520 |
+
"""الحصول على الكلمات في فئة معينة.
|
| 521 |
+
|
| 522 |
+
Args:
|
| 523 |
+
category: اسم الفئة.
|
| 524 |
+
|
| 525 |
+
Returns:
|
| 526 |
+
قائمة الكلمات (مرتبة).
|
| 527 |
+
"""
|
| 528 |
+
if category in self.categories:
|
| 529 |
+
return sorted(self.categories[category])
|
| 530 |
+
elif category == "custom_user":
|
| 531 |
+
return sorted(self.custom_vocabulary.keys())
|
| 532 |
+
return []
|
| 533 |
+
|
| 534 |
+
def search(self, query: str, max_results: int = 20) -> list[dict[str, str]]:
|
| 535 |
+
"""البحث في الكلمات المحمية.
|
| 536 |
+
|
| 537 |
+
Args:
|
| 538 |
+
query: نص البحث (يبحث في البداية — prefix search).
|
| 539 |
+
max_results: أقصى عدد نتائج.
|
| 540 |
+
|
| 541 |
+
Returns:
|
| 542 |
+
قائمة قواميس {"word": ..., "category": ...}.
|
| 543 |
+
"""
|
| 544 |
+
query_lower = query.lower()
|
| 545 |
+
results: list[dict[str, str]] = []
|
| 546 |
+
|
| 547 |
+
for cat_name, words in self.categories.items():
|
| 548 |
+
for word in words:
|
| 549 |
+
if word.lower().startswith(query_lower):
|
| 550 |
+
results.append({"word": word, "category": cat_name})
|
| 551 |
+
if len(results) >= max_results:
|
| 552 |
+
return results
|
| 553 |
+
|
| 554 |
+
for word, cat in self.custom_vocabulary.items():
|
| 555 |
+
if word.lower().startswith(query_lower):
|
| 556 |
+
results.append({"word": word, "category": cat})
|
| 557 |
+
if len(results) >= max_results:
|
| 558 |
+
return results
|
| 559 |
+
|
| 560 |
+
return results
|
| 561 |
+
|
| 562 |
+
# ------------------------------------------------------------------
|
| 563 |
+
# حفظ وتحميل المصطلحات المخصصة
|
| 564 |
+
# ------------------------------------------------------------------
|
| 565 |
+
|
| 566 |
+
def save_custom_vocabulary(self, path: str | Path) -> str:
|
| 567 |
+
"""حفظ المصطلحات المخصصة في ملف JSON.
|
| 568 |
+
|
| 569 |
+
Args:
|
| 570 |
+
path: مسار ملف الحفظ.
|
| 571 |
+
|
| 572 |
+
Returns:
|
| 573 |
+
مسار الملف المحفوظ.
|
| 574 |
+
"""
|
| 575 |
+
path = Path(path)
|
| 576 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 577 |
+
|
| 578 |
+
data = {
|
| 579 |
+
"custom_vocabulary": dict(self.custom_vocabulary),
|
| 580 |
+
"metadata": {
|
| 581 |
+
"total_words": len(self.custom_vocabulary),
|
| 582 |
+
"saved_at": self._get_timestamp(),
|
| 583 |
+
},
|
| 584 |
+
}
|
| 585 |
+
|
| 586 |
+
with open(path, "w", encoding="utf-8") as f:
|
| 587 |
+
json.dump(data, f, ensure_ascii=False, indent=2)
|
| 588 |
+
|
| 589 |
+
logger.info("تم حفظ %d مصطلح مخصص في: %s", len(self.custom_vocabulary), path)
|
| 590 |
+
return str(path)
|
| 591 |
+
|
| 592 |
+
def load_custom_vocabulary(self, path: str | Path) -> int:
|
| 593 |
+
"""تحميل المصطلحات المخصصة من ملف JSON.
|
| 594 |
+
|
| 595 |
+
Args:
|
| 596 |
+
path: مسار ملف التحميل.
|
| 597 |
+
|
| 598 |
+
Returns:
|
| 599 |
+
عدد المصطلحات المُحمَّلة.
|
| 600 |
+
"""
|
| 601 |
+
path = Path(path)
|
| 602 |
+
if not path.exists():
|
| 603 |
+
logger.warning("ملف المصطلحات غير موجود: %s", path)
|
| 604 |
+
return 0
|
| 605 |
+
|
| 606 |
+
try:
|
| 607 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 608 |
+
data = json.load(f)
|
| 609 |
+
|
| 610 |
+
vocab = data.get("custom_vocabulary", {})
|
| 611 |
+
if isinstance(vocab, dict):
|
| 612 |
+
count = 0
|
| 613 |
+
for word, category in vocab.items():
|
| 614 |
+
word = str(word).strip()
|
| 615 |
+
category = str(category).strip()
|
| 616 |
+
if word and word.lower() not in self._protected_index:
|
| 617 |
+
self.custom_vocabulary[word] = category
|
| 618 |
+
count += 1
|
| 619 |
+
|
| 620 |
+
self._rebuild_index()
|
| 621 |
+
logger.info("تم تحميل %d مصطلح مخصص من: %s", count, path)
|
| 622 |
+
return count
|
| 623 |
+
else:
|
| 624 |
+
logger.warning("صيغة ملف المصطلحات غير صحيحة: %s", path)
|
| 625 |
+
return 0
|
| 626 |
+
|
| 627 |
+
except json.JSONDecodeError as e:
|
| 628 |
+
logger.error("خطأ في قراءة ملف المصطلحات: %s", e)
|
| 629 |
+
return 0
|
| 630 |
+
except Exception as e:
|
| 631 |
+
logger.error("فشل في تحميل المصطلحات: %s", e)
|
| 632 |
+
return 0
|
| 633 |
+
|
| 634 |
+
@staticmethod
|
| 635 |
+
def _get_timestamp() -> str:
|
| 636 |
+
"""الحصول على الطابع الزمني الحالي."""
|
| 637 |
+
from datetime import datetime
|
| 638 |
+
return datetime.now().isoformat()
|
| 639 |
+
|
| 640 |
+
# ------------------------------------------------------------------
|
| 641 |
+
# تمثيل النص
|
| 642 |
+
# ------------------------------------------------------------------
|
| 643 |
+
|
| 644 |
+
def __repr__(self) -> str:
|
| 645 |
+
return (
|
| 646 |
+
f"ProtectedWordsManager("
|
| 647 |
+
f"total={len(self._protected_index)}, "
|
| 648 |
+
f"categories={len(self.categories)}, "
|
| 649 |
+
f"custom={len(self.custom_vocabulary)})"
|
| 650 |
+
)
|
| 651 |
+
|
| 652 |
+
def __len__(self) -> int:
|
| 653 |
+
return len(self._protected_index)
|
|
@@ -0,0 +1,1070 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
مولّد المرجع الدراسي — Study Guide Generator Module
|
| 3 |
+
======================================================
|
| 4 |
+
مولّد شامل للمراجع الدراسية من نتائج OCR.
|
| 5 |
+
|
| 6 |
+
القدرات:
|
| 7 |
+
- توليد مراجع Markdown منظمة من نتائج OCR
|
| 8 |
+
- مخططات Mermaid (خريطة ذهنية، مخطط انسيابي) من المفردات
|
| 9 |
+
- بطاقات Anki متوافقة (ثنائية اللغة، مفاهيمية، ملء فراغ)
|
| 10 |
+
- تصدير HTML أنيق مع دعم RTL و CSS للطباعة
|
| 11 |
+
- تعقيم التشكيل العربي لمعرّفات Mermaid صالحة
|
| 12 |
+
- تلوين المصطلحات البرمجية في Markdown
|
| 13 |
+
|
| 14 |
+
مثال الاستخدام:
|
| 15 |
+
>>> generator = StudyGuideGenerator()
|
| 16 |
+
>>> # من بيانات OCR مباشرة
|
| 17 |
+
>>> ocr_data = [
|
| 18 |
+
... {"text": "Python is great", "confidence": 0.95, "page": 1},
|
| 19 |
+
... {"text": "الثعبان رائع", "confidence": 0.88, "page": 1},
|
| 20 |
+
... ]
|
| 21 |
+
>>> md = generator.generate_markdown(ocr_data, title="My Notes")
|
| 22 |
+
>>> # مخطط Mermaid
|
| 23 |
+
>>> mermaid = generator.generate_mermaid(vocab, diagram_type="mindmap")
|
| 24 |
+
>>> # بطاقات Anki
|
| 25 |
+
>>> cards = generator.generate_flashcards(vocab, card_type="bilingual")
|
| 26 |
+
>>> generator.export_anki(cards, "flashcards.csv")
|
| 27 |
+
>>> # HTML
|
| 28 |
+
>>> generator.export_html(md_content, "guide.html")
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import csv
|
| 34 |
+
import json
|
| 35 |
+
import logging
|
| 36 |
+
import os
|
| 37 |
+
import random
|
| 38 |
+
from collections import defaultdict
|
| 39 |
+
from datetime import datetime
|
| 40 |
+
from pathlib import Path
|
| 41 |
+
from typing import Any, Optional
|
| 42 |
+
|
| 43 |
+
logger = logging.getLogger(__name__)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# ======================================================================
|
| 47 |
+
# أدوات مساعدة
|
| 48 |
+
# ======================================================================
|
| 49 |
+
|
| 50 |
+
# ألوان التلوين للمصطلحات البرمجية
|
| 51 |
+
SYNTAX_COLORS = {
|
| 52 |
+
"keywords": "#2563EB",
|
| 53 |
+
"types": "#16A34A",
|
| 54 |
+
"functions": "#9333EA",
|
| 55 |
+
"numbers": "#DC2626",
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
PYTHON_BUILTINS = {
|
| 59 |
+
"print", "input", "len", "range", "type", "int", "str", "float",
|
| 60 |
+
"list", "dict", "set", "tuple", "bool", "open", "super", "self",
|
| 61 |
+
"append", "extend", "pop", "sort", "join", "split", "strip",
|
| 62 |
+
"enumerate", "zip", "map", "filter", "sorted", "reversed",
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
DATA_TYPES = {
|
| 66 |
+
"integers", "floating", "points", "strings", "boolean", "list",
|
| 67 |
+
"dictionary", "tuple", "none", "mutable", "immutable",
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def sanitize_mermaid_id(text: str) -> str:
|
| 72 |
+
"""تعقيم النص ليكون معرّفاً صالحاً في Mermaid.
|
| 73 |
+
|
| 74 |
+
يزيل التشكيل العربي والرموز الخاصة ويضمن بدءاً بحرف.
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
text: النص الأصلي.
|
| 78 |
+
|
| 79 |
+
Returns:
|
| 80 |
+
معرّف Mermaid صالح (بحد أقصى 40 حرفاً).
|
| 81 |
+
"""
|
| 82 |
+
clean = text.strip()
|
| 83 |
+
|
| 84 |
+
# استبدال الرموز غير المسموحة
|
| 85 |
+
for ch in ['"', "'", '(', ')', '{', '}', '[', ']', '<', '>',
|
| 86 |
+
'/', '\\', '&', '#', '|', ';', ':', ',', '.', '!', '?']:
|
| 87 |
+
clean = clean.replace(ch, "_")
|
| 88 |
+
|
| 89 |
+
# إزالة التشكيل العربي (Tashkeel / Diacritics)
|
| 90 |
+
arabic_diacritics = set(
|
| 91 |
+
"\u0610\u0611\u0612\u0613\u0614\u0615\u0616\u0617\u0618\u0619\u061A"
|
| 92 |
+
"\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652"
|
| 93 |
+
)
|
| 94 |
+
clean = "".join(c for c in clean if c not in arabic_diacritics)
|
| 95 |
+
|
| 96 |
+
# استبدال المسافات
|
| 97 |
+
clean = clean.replace(" ", "_")
|
| 98 |
+
|
| 99 |
+
# التأكد من أن المعرّف ليس فارغاً
|
| 100 |
+
if not clean:
|
| 101 |
+
clean = "term"
|
| 102 |
+
|
| 103 |
+
# لا يبدأ برقم
|
| 104 |
+
if clean[0].isdigit():
|
| 105 |
+
clean = "t_" + clean
|
| 106 |
+
|
| 107 |
+
# حد أقصى للطول
|
| 108 |
+
return clean[:40]
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def highlight_python_terms(text: str) -> str:
|
| 112 |
+
"""تلوين المصطلحات البرمجية في النص (HTML spans).
|
| 113 |
+
|
| 114 |
+
Args:
|
| 115 |
+
text: النص الأصلي.
|
| 116 |
+
|
| 117 |
+
Returns:
|
| 118 |
+
النص مع HTML spans للتلوين.
|
| 119 |
+
"""
|
| 120 |
+
if not text:
|
| 121 |
+
return text
|
| 122 |
+
|
| 123 |
+
words = text.split()
|
| 124 |
+
highlighted = []
|
| 125 |
+
|
| 126 |
+
for word in words:
|
| 127 |
+
clean = word.strip(".,;:!?\"'()-*").lower()
|
| 128 |
+
|
| 129 |
+
if clean in PYTHON_BUILTINS:
|
| 130 |
+
highlighted.append(
|
| 131 |
+
f'<span style="color:{SYNTAX_COLORS["functions"]}">{word}</span>'
|
| 132 |
+
)
|
| 133 |
+
elif clean in DATA_TYPES:
|
| 134 |
+
highlighted.append(
|
| 135 |
+
f'<span style="color:{SYNTAX_COLORS["types"]}">{word}</span>'
|
| 136 |
+
)
|
| 137 |
+
else:
|
| 138 |
+
highlighted.append(word)
|
| 139 |
+
|
| 140 |
+
return " ".join(highlighted)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def detect_language_simple(text: str) -> str:
|
| 144 |
+
"""كشف لغة النص ببساطة (عربي/إنجليزي/مختلط).
|
| 145 |
+
|
| 146 |
+
Args:
|
| 147 |
+
text: النص المراد فحصه.
|
| 148 |
+
|
| 149 |
+
Returns:
|
| 150 |
+
"ar", "en", أو "mixed".
|
| 151 |
+
"""
|
| 152 |
+
if not text:
|
| 153 |
+
return "unknown"
|
| 154 |
+
|
| 155 |
+
arabic_chars = sum(1 for c in text if "\u0600" <= c <= "\u06FF")
|
| 156 |
+
latin_chars = sum(1 for c in text if c.isascii() and c.isalpha())
|
| 157 |
+
|
| 158 |
+
if arabic_chars == 0 and latin_chars > 0:
|
| 159 |
+
return "en"
|
| 160 |
+
if latin_chars == 0 and arabic_chars > 0:
|
| 161 |
+
return "ar"
|
| 162 |
+
if arabic_chars > 0 and latin_chars > 0:
|
| 163 |
+
return "mixed"
|
| 164 |
+
return "unknown"
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
# ======================================================================
|
| 168 |
+
# فئة مولّد المرجع الدراسي
|
| 169 |
+
# ======================================================================
|
| 170 |
+
|
| 171 |
+
class StudyGuideGenerator:
|
| 172 |
+
"""مولّد شامل للمراجع الدراسية من نتائج OCR.
|
| 173 |
+
|
| 174 |
+
يدعم:
|
| 175 |
+
- Markdown مع تلوين المصطلحات
|
| 176 |
+
- مخططات Mermaid (mindmap, flowchart)
|
| 177 |
+
- بطاقات Anki متوافقة
|
| 178 |
+
- HTML مع RTL و CSS للطباعة
|
| 179 |
+
|
| 180 |
+
Attributes:
|
| 181 |
+
highlight_terms: تفعيل تلوين المصطلحات البرمجية.
|
| 182 |
+
"""
|
| 183 |
+
|
| 184 |
+
def __init__(
|
| 185 |
+
self,
|
| 186 |
+
highlight_terms: bool = True,
|
| 187 |
+
default_language: str = "ar",
|
| 188 |
+
) -> None:
|
| 189 |
+
"""تهيئة مولّد المرجع الدراسي.
|
| 190 |
+
|
| 191 |
+
Args:
|
| 192 |
+
highlight_terms: تفعيل تلوين المصطلحات البرمجية في Markdown.
|
| 193 |
+
default_language: اللغة الافتراضية ("ar" أو "en").
|
| 194 |
+
"""
|
| 195 |
+
self.highlight_terms = highlight_terms
|
| 196 |
+
self.default_language = default_language
|
| 197 |
+
|
| 198 |
+
# ------------------------------------------------------------------
|
| 199 |
+
# توليد Markdown
|
| 200 |
+
# ------------------------------------------------------------------
|
| 201 |
+
|
| 202 |
+
def generate_markdown(
|
| 203 |
+
self,
|
| 204 |
+
ocr_results: list[dict[str, Any]],
|
| 205 |
+
title: str = "مرجع دراسة — مستخرج من الملاحظات",
|
| 206 |
+
highlight: Optional[bool] = None,
|
| 207 |
+
page_grouping: bool = True,
|
| 208 |
+
) -> str:
|
| 209 |
+
"""توليد مرجع دراسي بصيغة Markdown من نتائج OCR.
|
| 210 |
+
|
| 211 |
+
Args:
|
| 212 |
+
ocr_results: قائمة نتائج OCR، كل عنصر قاموس يحتوي:
|
| 213 |
+
- text: النص المستخرج
|
| 214 |
+
- confidence: الثقة (اختياري)
|
| 215 |
+
- page: رقم الصفحة (اختياري)
|
| 216 |
+
- source: المحرك (اختياري)
|
| 217 |
+
title: عنوان المرجع.
|
| 218 |
+
highlight: تفعيل التلوين (None = استخدام الإعداد الافتراضي).
|
| 219 |
+
page_grouping: تجميع النتائج حسب الصفحات.
|
| 220 |
+
|
| 221 |
+
Returns:
|
| 222 |
+
محتوى Markdown الكامل.
|
| 223 |
+
|
| 224 |
+
Example:
|
| 225 |
+
>>> results = [
|
| 226 |
+
... {"text": "Python programming", "page": 1, "confidence": 0.95},
|
| 227 |
+
... {"text": "متغيرات", "page": 1, "confidence": 0.88},
|
| 228 |
+
... ]
|
| 229 |
+
>>> md = generator.generate_markdown(results)
|
| 230 |
+
"""
|
| 231 |
+
if not ocr_results:
|
| 232 |
+
logger.warning("لا توجد بيانات OCR لتوليد المرجع الدراسي")
|
| 233 |
+
return ""
|
| 234 |
+
|
| 235 |
+
if highlight is None:
|
| 236 |
+
highlight = self.highlight_terms
|
| 237 |
+
|
| 238 |
+
lines: list[str] = []
|
| 239 |
+
lines.append(f"# {title}\n")
|
| 240 |
+
lines.append(f"تاريخ الإنشاء: {datetime.now().strftime('%Y-%m-%d %H:%M')}\n")
|
| 241 |
+
lines.append(f"عدد النتائج: {len(ocr_results)}\n")
|
| 242 |
+
lines.append("---\n")
|
| 243 |
+
|
| 244 |
+
if page_grouping:
|
| 245 |
+
# تجميع حسب الصفحات
|
| 246 |
+
pages: dict[Any, list[dict]] = defaultdict(list)
|
| 247 |
+
for item in ocr_results:
|
| 248 |
+
page = item.get("page", "غير محدد")
|
| 249 |
+
pages[page].append(item)
|
| 250 |
+
|
| 251 |
+
for page_num in sorted(
|
| 252 |
+
pages.keys(),
|
| 253 |
+
key=lambda p: int(p) if isinstance(p, (int, float)) else 0,
|
| 254 |
+
):
|
| 255 |
+
items = pages[page_num]
|
| 256 |
+
lines.append(f"\n## صفحة رقم: {page_num}\n")
|
| 257 |
+
|
| 258 |
+
# استخراج المصطلحات (أزواج إنجليزي-عربي)
|
| 259 |
+
vocab = self._extract_vocabulary_from_items(items)
|
| 260 |
+
if vocab:
|
| 261 |
+
lines.append("### المصطلحات\n")
|
| 262 |
+
lines.append("| الإنجليزية | العربية |")
|
| 263 |
+
lines.append("| --- | --- |")
|
| 264 |
+
for v in vocab:
|
| 265 |
+
en = v.get("english", "").strip()
|
| 266 |
+
ar = v.get("arabic", "").strip()
|
| 267 |
+
if en or ar:
|
| 268 |
+
lines.append(f"| {en} | {ar} |")
|
| 269 |
+
lines.append("")
|
| 270 |
+
|
| 271 |
+
# الملاحظات
|
| 272 |
+
text_items = [i for i in items if i.get("text", "").strip()]
|
| 273 |
+
if text_items:
|
| 274 |
+
lines.append("### الملاحظات والشروحات\n")
|
| 275 |
+
for item in text_items:
|
| 276 |
+
text = str(item["text"]).strip()
|
| 277 |
+
conf = item.get("confidence", 0)
|
| 278 |
+
|
| 279 |
+
if highlight:
|
| 280 |
+
text = highlight_python_terms(text)
|
| 281 |
+
|
| 282 |
+
lang_indicator = detect_language_simple(text)
|
| 283 |
+
conf_str = f" *(ثقة: {conf:.0%})*" if conf else ""
|
| 284 |
+
|
| 285 |
+
if lang_indicator == "ar":
|
| 286 |
+
lines.append(f"- {text}{conf_str}")
|
| 287 |
+
else:
|
| 288 |
+
lines.append(f"- {text} *(EN)*{conf_str}")
|
| 289 |
+
|
| 290 |
+
lines.append("")
|
| 291 |
+
else:
|
| 292 |
+
# بدون تجميع حسب الصفحات
|
| 293 |
+
lines.append("\n## الملاحظات\n")
|
| 294 |
+
for item in ocr_results:
|
| 295 |
+
text = str(item.get("text", "")).strip()
|
| 296 |
+
if not text:
|
| 297 |
+
continue
|
| 298 |
+
|
| 299 |
+
if highlight:
|
| 300 |
+
text = highlight_python_terms(text)
|
| 301 |
+
|
| 302 |
+
lang = detect_language_simple(text)
|
| 303 |
+
if lang == "ar":
|
| 304 |
+
lines.append(f"- {text}")
|
| 305 |
+
else:
|
| 306 |
+
lines.append(f"- {text} *(EN)*")
|
| 307 |
+
|
| 308 |
+
lines.append("")
|
| 309 |
+
|
| 310 |
+
content = "\n".join(lines)
|
| 311 |
+
logger.info("تم توليد مرجع Markdown: %d سطر", len(lines))
|
| 312 |
+
return content
|
| 313 |
+
|
| 314 |
+
# ------------------------------------------------------------------
|
| 315 |
+
# مخططات Mermaid
|
| 316 |
+
# ------------------------------------------------------------------
|
| 317 |
+
|
| 318 |
+
def generate_mermaid(
|
| 319 |
+
self,
|
| 320 |
+
vocabulary: list[dict[str, str]],
|
| 321 |
+
diagram_type: str = "mindmap",
|
| 322 |
+
max_terms: int = 50,
|
| 323 |
+
title: str = "المصطلحات",
|
| 324 |
+
) -> str:
|
| 325 |
+
"""توليد مخطط Mermaid من المفردات.
|
| 326 |
+
|
| 327 |
+
Args:
|
| 328 |
+
vocabulary: قائمة أزواج {"english": ..., "arabic": ...}.
|
| 329 |
+
diagram_type: نوع المخطط ("mindmap", "flowchart").
|
| 330 |
+
max_terms: أقصى عدد مصطلحات.
|
| 331 |
+
title: عنوان المخطط.
|
| 332 |
+
|
| 333 |
+
Returns:
|
| 334 |
+
نص Mermaid جاهز للتضمين.
|
| 335 |
+
|
| 336 |
+
Example:
|
| 337 |
+
>>> vocab = [
|
| 338 |
+
... {"english": "variable", "arabic": "متغير"},
|
| 339 |
+
... {"english": "function", "arabic": "دالة"},
|
| 340 |
+
... ]
|
| 341 |
+
>>> mermaid = generator.generate_mermaid(vocab, diagram_type="mindmap")
|
| 342 |
+
>>> print(f"```mermaid\\n{mermaid}\\n```")
|
| 343 |
+
"""
|
| 344 |
+
if not vocabulary:
|
| 345 |
+
logger.warning("لا توجد مفردات لتوليد مخطط Mermaid")
|
| 346 |
+
return ""
|
| 347 |
+
|
| 348 |
+
vocab = vocabulary[:max_terms]
|
| 349 |
+
|
| 350 |
+
if diagram_type == "mindmap":
|
| 351 |
+
return self._generate_mindmap(vocab, title)
|
| 352 |
+
elif diagram_type == "flowchart":
|
| 353 |
+
return self._generate_flowchart(vocab, title)
|
| 354 |
+
else:
|
| 355 |
+
logger.warning("نوع مخطط غير مدعوم: '%s' — استخدام mindmap", diagram_type)
|
| 356 |
+
return self._generate_mindmap(vocab, title)
|
| 357 |
+
|
| 358 |
+
def _generate_mindmap(
|
| 359 |
+
self,
|
| 360 |
+
vocab: list[dict[str, str]],
|
| 361 |
+
title: str,
|
| 362 |
+
) -> str:
|
| 363 |
+
"""توليد خريطة ذهنية Mermaid."""
|
| 364 |
+
lines = ["mindmap", f" root(({title}))"]
|
| 365 |
+
|
| 366 |
+
for v in vocab:
|
| 367 |
+
en = v.get("english", "").strip()
|
| 368 |
+
ar = v.get("arabic", "").strip()
|
| 369 |
+
|
| 370 |
+
if en and ar:
|
| 371 |
+
en_id = sanitize_mermaid_id(en)
|
| 372 |
+
ar_id = sanitize_mermaid_id(ar)
|
| 373 |
+
lines.append(f" {en_id}[{en}]")
|
| 374 |
+
lines.append(f" {ar_id}[{ar}]")
|
| 375 |
+
elif en:
|
| 376 |
+
en_id = sanitize_mermaid_id(en)
|
| 377 |
+
lines.append(f" {en_id}[{en}]")
|
| 378 |
+
elif ar:
|
| 379 |
+
ar_id = sanitize_mermaid_id(ar)
|
| 380 |
+
lines.append(f" {ar_id}[{ar}]")
|
| 381 |
+
|
| 382 |
+
return "\n".join(lines)
|
| 383 |
+
|
| 384 |
+
def _generate_flowchart(
|
| 385 |
+
self,
|
| 386 |
+
vocab: list[dict[str, str]],
|
| 387 |
+
title: str,
|
| 388 |
+
) -> str:
|
| 389 |
+
"""توليد مخطط انسيابي Mermaid."""
|
| 390 |
+
lines = ["flowchart LR"]
|
| 391 |
+
root_id = sanitize_mermaid_id(title)
|
| 392 |
+
lines.append(f" {root_id}[{title}]")
|
| 393 |
+
|
| 394 |
+
for v in vocab:
|
| 395 |
+
en = v.get("english", "").strip()
|
| 396 |
+
ar = v.get("arabic", "").strip()
|
| 397 |
+
|
| 398 |
+
if en and ar:
|
| 399 |
+
en_id = sanitize_mermaid_id(en)
|
| 400 |
+
ar_id = sanitize_mermaid_id(ar)
|
| 401 |
+
label = f"{en} = {ar}"
|
| 402 |
+
term_id = sanitize_mermaid_id(label)
|
| 403 |
+
lines.append(f" {root_id} --> {term_id}[{label}]")
|
| 404 |
+
elif en:
|
| 405 |
+
en_id = sanitize_mermaid_id(en)
|
| 406 |
+
lines.append(f" {root_id} --> {en_id}[{en}]")
|
| 407 |
+
elif ar:
|
| 408 |
+
ar_id = sanitize_mermaid_id(ar)
|
| 409 |
+
lines.append(f" {root_id} --> {ar_id}[{ar}]")
|
| 410 |
+
|
| 411 |
+
return "\n".join(lines)
|
| 412 |
+
|
| 413 |
+
# ------------------------------------------------------------------
|
| 414 |
+
# بطاقات تعليمية (Flashcards)
|
| 415 |
+
# ------------------------------------------------------------------
|
| 416 |
+
|
| 417 |
+
def generate_flashcards(
|
| 418 |
+
self,
|
| 419 |
+
vocabulary: list[dict[str, str]],
|
| 420 |
+
card_type: str = "bilingual",
|
| 421 |
+
max_cards: int = 200,
|
| 422 |
+
shuffle: bool = True,
|
| 423 |
+
) -> list[dict[str, Any]]:
|
| 424 |
+
"""توليد بطاقات تعليمية من المفردات.
|
| 425 |
+
|
| 426 |
+
أنواع البطاقات:
|
| 427 |
+
- "bilingual": ثنائية اللغة (وجه إنجليزي / ظهر عربي والعكس)
|
| 428 |
+
- "concept": مفاهيمية (مصطلح / ترجمة)
|
| 429 |
+
- "fill_blank": ملء فراغ (جملة مع كلمة محجوبة)
|
| 430 |
+
|
| 431 |
+
Args:
|
| 432 |
+
vocabulary: قائمة أزواج {"english": ..., "arabic": ...}.
|
| 433 |
+
card_type: نوع البطاقات.
|
| 434 |
+
max_cards: أقصى عدد بطاقات.
|
| 435 |
+
shuffle: خلط البطاقات.
|
| 436 |
+
|
| 437 |
+
Returns:
|
| 438 |
+
قائمة بطاقات: [{"front": ..., "back": ..., "tags": ...}].
|
| 439 |
+
|
| 440 |
+
Example:
|
| 441 |
+
>>> vocab = [
|
| 442 |
+
... {"english": "variable", "arabic": "متغير"},
|
| 443 |
+
... {"english": "loop", "arabic": "حلقة"},
|
| 444 |
+
... ]
|
| 445 |
+
>>> cards = generator.generate_flashcards(vocab, card_type="bilingual")
|
| 446 |
+
"""
|
| 447 |
+
if not vocabulary:
|
| 448 |
+
logger.warning("لا توجد مفردات لتوليد بطاقات تعليمية")
|
| 449 |
+
return []
|
| 450 |
+
|
| 451 |
+
if card_type == "bilingual":
|
| 452 |
+
cards = self._generate_bilingual_cards(vocabulary)
|
| 453 |
+
elif card_type == "concept":
|
| 454 |
+
cards = self._generate_concept_cards(vocabulary)
|
| 455 |
+
elif card_type == "fill_blank":
|
| 456 |
+
cards = self._generate_fill_blank_cards(vocabulary)
|
| 457 |
+
else:
|
| 458 |
+
logger.warning("نوع بطاقات غير مدعوم: '%s' — استخدام bilingual", card_type)
|
| 459 |
+
cards = self._generate_bilingual_cards(vocabulary)
|
| 460 |
+
|
| 461 |
+
if shuffle:
|
| 462 |
+
random.shuffle(cards)
|
| 463 |
+
|
| 464 |
+
return cards[:max_cards]
|
| 465 |
+
|
| 466 |
+
@staticmethod
|
| 467 |
+
def _generate_bilingual_cards(vocab: list[dict[str, str]]) -> list[dict[str, Any]]:
|
| 468 |
+
"""بطاقات ثنائية اللغة: EN→AR و AR→EN."""
|
| 469 |
+
cards = []
|
| 470 |
+
|
| 471 |
+
for v in vocab:
|
| 472 |
+
en = v.get("english", "").strip()
|
| 473 |
+
ar = v.get("arabic", "").strip()
|
| 474 |
+
|
| 475 |
+
if en and ar:
|
| 476 |
+
# بطاقة EN → AR
|
| 477 |
+
cards.append({
|
| 478 |
+
"front": en,
|
| 479 |
+
"back": ar,
|
| 480 |
+
"tags": ["EN-AR", "bilingual"],
|
| 481 |
+
})
|
| 482 |
+
# بطاقة AR → EN
|
| 483 |
+
cards.append({
|
| 484 |
+
"front": ar,
|
| 485 |
+
"back": en,
|
| 486 |
+
"tags": ["AR-EN", "bilingual"],
|
| 487 |
+
})
|
| 488 |
+
|
| 489 |
+
return cards
|
| 490 |
+
|
| 491 |
+
@staticmethod
|
| 492 |
+
def _generate_concept_cards(vocab: list[dict[str, str]]) -> list[dict[str, Any]]:
|
| 493 |
+
"""بطاقات مفاهيمية: المصطلح والترجمة."""
|
| 494 |
+
cards = []
|
| 495 |
+
|
| 496 |
+
for v in vocab:
|
| 497 |
+
en = v.get("english", "").strip()
|
| 498 |
+
ar = v.get("arabic", "").strip()
|
| 499 |
+
|
| 500 |
+
if en:
|
| 501 |
+
cards.append({
|
| 502 |
+
"front": en,
|
| 503 |
+
"back": f"الترجمة العربية: {ar}" if ar else "(بدون ترجمة)",
|
| 504 |
+
"tags": ["concept"],
|
| 505 |
+
})
|
| 506 |
+
|
| 507 |
+
if ar and en:
|
| 508 |
+
cards.append({
|
| 509 |
+
"front": ar,
|
| 510 |
+
"back": f"English: {en}",
|
| 511 |
+
"tags": ["concept"],
|
| 512 |
+
})
|
| 513 |
+
|
| 514 |
+
return cards
|
| 515 |
+
|
| 516 |
+
@staticmethod
|
| 517 |
+
def _generate_fill_blank_cards(vocab: list[dict[str, str]]) -> list[dict[str, Any]]:
|
| 518 |
+
"""بطاقات ملء فراغ: جملة بسيطة مع كلمة محجوبة."""
|
| 519 |
+
cards = []
|
| 520 |
+
|
| 521 |
+
for v in vocab:
|
| 522 |
+
en = v.get("english", "").strip()
|
| 523 |
+
ar = v.get("arabic", "").strip()
|
| 524 |
+
|
| 525 |
+
if en:
|
| 526 |
+
# جملة بسيطة: "The term ___ means ..."
|
| 527 |
+
cards.append({
|
| 528 |
+
"front": f"The term ___ in English refers to: {ar}" if ar else f"Fill in: ___",
|
| 529 |
+
"back": en,
|
| 530 |
+
"tags": ["fill_blank", "EN"],
|
| 531 |
+
})
|
| 532 |
+
|
| 533 |
+
if ar:
|
| 534 |
+
cards.append({
|
| 535 |
+
"front": f"المصطلح ___ بالإنجليزية يعني: {en}" if en else "أكمل: ___",
|
| 536 |
+
"back": ar,
|
| 537 |
+
"tags": ["fill_blank", "AR"],
|
| 538 |
+
})
|
| 539 |
+
|
| 540 |
+
return cards
|
| 541 |
+
|
| 542 |
+
# ------------------------------------------------------------------
|
| 543 |
+
# تصدير Anki
|
| 544 |
+
# ------------------------------------------------------------------
|
| 545 |
+
|
| 546 |
+
def export_anki(
|
| 547 |
+
self,
|
| 548 |
+
cards: list[dict[str, Any]],
|
| 549 |
+
output_path: str | Path,
|
| 550 |
+
deck_name: str = "OmniFile::Study",
|
| 551 |
+
include_tags: bool = True,
|
| 552 |
+
) -> str:
|
| 553 |
+
"""تصدير بطاقات تعليمية بتنسيق CSV متوافق مع Anki.
|
| 554 |
+
|
| 555 |
+
تنسيق Anki: front;back;tags (مع فاصل منقوطة).
|
| 556 |
+
|
| 557 |
+
Args:
|
| 558 |
+
cards: قائمة البطاقات من generate_flashcards().
|
| 559 |
+
output_path: مسار حفظ ملف CSV.
|
| 560 |
+
deck_name: اسم ال��اقة في Anki (يُكتب في التعليق).
|
| 561 |
+
include_tags: تضمين الوسوم.
|
| 562 |
+
|
| 563 |
+
Returns:
|
| 564 |
+
مسار الملف المحفوظ.
|
| 565 |
+
|
| 566 |
+
Example:
|
| 567 |
+
>>> generator.export_anki(cards, "my_flashcards.csv")
|
| 568 |
+
>>> # استيراد في Anki: File > Import > نوع: "Separated by Semicolon"
|
| 569 |
+
"""
|
| 570 |
+
if not cards:
|
| 571 |
+
logger.warning("لا توجد بطاقات للتصدير")
|
| 572 |
+
return ""
|
| 573 |
+
|
| 574 |
+
output_path = Path(output_path)
|
| 575 |
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 576 |
+
|
| 577 |
+
with open(output_path, "w", encoding="utf-8-sig", newline="") as f:
|
| 578 |
+
writer = csv.writer(f, delimiter=";")
|
| 579 |
+
|
| 580 |
+
# رأس مع تعليق اسم الباقة
|
| 581 |
+
if include_tags:
|
| 582 |
+
writer.writerow(["front", "back", "tags"])
|
| 583 |
+
else:
|
| 584 |
+
writer.writerow(["front", "back"])
|
| 585 |
+
|
| 586 |
+
for card in cards:
|
| 587 |
+
front = str(card.get("front", "")).replace(";", ",").replace("\n", " ")
|
| 588 |
+
back = str(card.get("back", "")).replace(";", ",").replace("\n", " ")
|
| 589 |
+
|
| 590 |
+
if include_tags:
|
| 591 |
+
tags_str = " ".join(card.get("tags", []))
|
| 592 |
+
writer.writerow([front, back, tags_str])
|
| 593 |
+
else:
|
| 594 |
+
writer.writerow([front, back])
|
| 595 |
+
|
| 596 |
+
logger.info(
|
| 597 |
+
"تم تصدير %d بطاقة Anki إلى: %s (الباقة: %s)",
|
| 598 |
+
len(cards), output_path, deck_name,
|
| 599 |
+
)
|
| 600 |
+
return str(output_path)
|
| 601 |
+
|
| 602 |
+
# ------------------------------------------------------------------
|
| 603 |
+
# تصدير HTML
|
| 604 |
+
# ------------------------------------------------------------------
|
| 605 |
+
|
| 606 |
+
def export_html(
|
| 607 |
+
self,
|
| 608 |
+
markdown_content: str,
|
| 609 |
+
output_path: str | Path,
|
| 610 |
+
title: str = "مرجع دراسة",
|
| 611 |
+
include_print_css: bool = True,
|
| 612 |
+
) -> str:
|
| 613 |
+
"""تصدير المرجع إلى HTML أنيق مع دعم RTL و CSS للطباعة.
|
| 614 |
+
|
| 615 |
+
Args:
|
| 616 |
+
markdown_content: محتوى Markdown.
|
| 617 |
+
output_path: مسار حفظ ملف HTML.
|
| 618 |
+
title: عنوان الصفحة.
|
| 619 |
+
include_print_css: تضمين أنماط الطباعة.
|
| 620 |
+
|
| 621 |
+
Returns:
|
| 622 |
+
مسار الملف المحفوظ.
|
| 623 |
+
|
| 624 |
+
Example:
|
| 625 |
+
>>> generator.export_html(markdown_text, "study_guide.html")
|
| 626 |
+
"""
|
| 627 |
+
output_path = Path(output_path)
|
| 628 |
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 629 |
+
|
| 630 |
+
print_css = ""
|
| 631 |
+
if include_print_css:
|
| 632 |
+
print_css = """
|
| 633 |
+
@media print {
|
| 634 |
+
body { padding: 0; margin: 0; }
|
| 635 |
+
h2 { page-break-before: auto; }
|
| 636 |
+
table { page-break-inside: avoid; }
|
| 637 |
+
.no-print { display: none; }
|
| 638 |
+
}"""
|
| 639 |
+
|
| 640 |
+
html = f"""<!DOCTYPE html>
|
| 641 |
+
<html lang="ar" dir="rtl">
|
| 642 |
+
<head>
|
| 643 |
+
<meta charset="UTF-8">
|
| 644 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 645 |
+
<title>{title}</title>
|
| 646 |
+
<style>
|
| 647 |
+
/* === خطوط === */
|
| 648 |
+
@import url('https://fonts.googleapis.com/css2?family=Amiri:wght@400;700&family=Noto+Sans+Arabic:wght@400;700&display=swap');
|
| 649 |
+
|
| 650 |
+
/* === أساسيات === */
|
| 651 |
+
body {{
|
| 652 |
+
font-family: 'Amiri', 'Noto Sans Arabic', 'Segoe UI', sans-serif;
|
| 653 |
+
max-width: 900px;
|
| 654 |
+
margin: 0 auto;
|
| 655 |
+
padding: 30px;
|
| 656 |
+
line-height: 1.9;
|
| 657 |
+
color: #1a1a2e;
|
| 658 |
+
background: #ffffff;
|
| 659 |
+
direction: rtl;
|
| 660 |
+
}}
|
| 661 |
+
|
| 662 |
+
/* === عناوين === */
|
| 663 |
+
h1 {{
|
| 664 |
+
text-align: center;
|
| 665 |
+
color: #16213e;
|
| 666 |
+
border-bottom: 3px solid #0f3460;
|
| 667 |
+
padding-bottom: 15px;
|
| 668 |
+
font-size: 1.8em;
|
| 669 |
+
}}
|
| 670 |
+
h2 {{
|
| 671 |
+
color: #0f3460;
|
| 672 |
+
border-right: 4px solid #e94560;
|
| 673 |
+
padding-right: 15px;
|
| 674 |
+
margin-top: 2em;
|
| 675 |
+
font-size: 1.4em;
|
| 676 |
+
}}
|
| 677 |
+
h3 {{
|
| 678 |
+
color: #533483;
|
| 679 |
+
font-size: 1.2em;
|
| 680 |
+
}}
|
| 681 |
+
|
| 682 |
+
/* === جداول === */
|
| 683 |
+
table {{
|
| 684 |
+
border-collapse: collapse;
|
| 685 |
+
width: 100%;
|
| 686 |
+
margin: 15px 0;
|
| 687 |
+
font-size: 14px;
|
| 688 |
+
}}
|
| 689 |
+
th, td {{
|
| 690 |
+
border: 1px solid #ddd;
|
| 691 |
+
padding: 10px 15px;
|
| 692 |
+
text-align: right;
|
| 693 |
+
}}
|
| 694 |
+
th {{
|
| 695 |
+
background: #0f3460;
|
| 696 |
+
color: white;
|
| 697 |
+
font-weight: bold;
|
| 698 |
+
}}
|
| 699 |
+
tr:nth-child(even) {{
|
| 700 |
+
background: #f8f9fa;
|
| 701 |
+
}}
|
| 702 |
+
tr:hover {{
|
| 703 |
+
background: #e8f0fe;
|
| 704 |
+
}}
|
| 705 |
+
|
| 706 |
+
/* === كود === */
|
| 707 |
+
code {{
|
| 708 |
+
background: #f4f4f8;
|
| 709 |
+
padding: 2px 6px;
|
| 710 |
+
border-radius: 4px;
|
| 711 |
+
font-family: 'JetBrains Mono', 'Fira Code', monospace;
|
| 712 |
+
font-size: 0.9em;
|
| 713 |
+
color: #d63384;
|
| 714 |
+
}}
|
| 715 |
+
pre {{
|
| 716 |
+
background: #1e1e2e;
|
| 717 |
+
color: #cdd6f4;
|
| 718 |
+
padding: 15px;
|
| 719 |
+
border-radius: 8px;
|
| 720 |
+
overflow-x: auto;
|
| 721 |
+
direction: ltr;
|
| 722 |
+
text-align: left;
|
| 723 |
+
}}
|
| 724 |
+
pre code {{
|
| 725 |
+
background: none;
|
| 726 |
+
color: inherit;
|
| 727 |
+
padding: 0;
|
| 728 |
+
}}
|
| 729 |
+
|
| 730 |
+
/* === قوائم === */
|
| 731 |
+
ul, ol {{
|
| 732 |
+
padding-right: 25px;
|
| 733 |
+
}}
|
| 734 |
+
li {{
|
| 735 |
+
margin-bottom: 5px;
|
| 736 |
+
}}
|
| 737 |
+
|
| 738 |
+
/* === فاصل === */
|
| 739 |
+
hr {{
|
| 740 |
+
border: none;
|
| 741 |
+
height: 2px;
|
| 742 |
+
background: linear-gradient(to right, #0f3460, #e94560, #0f3460);
|
| 743 |
+
margin: 2em 0;
|
| 744 |
+
}}
|
| 745 |
+
|
| 746 |
+
/* === روابط === */
|
| 747 |
+
a {{
|
| 748 |
+
color: #2563eb;
|
| 749 |
+
text-decoration: none;
|
| 750 |
+
}}
|
| 751 |
+
a:hover {{
|
| 752 |
+
text-decoration: underline;
|
| 753 |
+
}}
|
| 754 |
+
|
| 755 |
+
/* === ملخص === */
|
| 756 |
+
.metadata {{
|
| 757 |
+
background: #f0f4ff;
|
| 758 |
+
border-right: 3px solid #2563eb;
|
| 759 |
+
padding: 10px 15px;
|
| 760 |
+
margin: 15px 0;
|
| 761 |
+
font-size: 0.9em;
|
| 762 |
+
color: #4a5568;
|
| 763 |
+
}}
|
| 764 |
+
|
| 765 |
+
/* === بطاقات === */
|
| 766 |
+
.flashcard {{
|
| 767 |
+
background: #fafafa;
|
| 768 |
+
border: 1px solid #e2e8f0;
|
| 769 |
+
border-radius: 8px;
|
| 770 |
+
padding: 15px;
|
| 771 |
+
margin: 10px 0;
|
| 772 |
+
}}
|
| 773 |
+
.flashcard-front {{
|
| 774 |
+
font-weight: bold;
|
| 775 |
+
color: #2d3748;
|
| 776 |
+
margin-bottom: 8px;
|
| 777 |
+
}}
|
| 778 |
+
.flashcard-back {{
|
| 779 |
+
color: #718096;
|
| 780 |
+
border-top: 1px dashed #cbd5e0;
|
| 781 |
+
padding-top: 8px;
|
| 782 |
+
}}
|
| 783 |
+
.flashcard-tags {{
|
| 784 |
+
font-size: 0.8em;
|
| 785 |
+
color: #a0aec0;
|
| 786 |
+
margin-top: 5px;
|
| 787 |
+
}}
|
| 788 |
+
|
| 789 |
+
/* === طباعة === */
|
| 790 |
+
{print_css}
|
| 791 |
+
</style>
|
| 792 |
+
</head>
|
| 793 |
+
<body>
|
| 794 |
+
"""
|
| 795 |
+
|
| 796 |
+
# تحويل Markdown إلى HTML
|
| 797 |
+
html += self._markdown_to_html(markdown_content)
|
| 798 |
+
|
| 799 |
+
html += f"""
|
| 800 |
+
<div class="no-print" style="text-align: center; margin-top: 30px; color: #999; font-size: 0.8em;">
|
| 801 |
+
<p>تم الإنشاء بواسطة OmniFile Processor — {datetime.now().strftime('%Y-%m-%d %H:%M')}</p>
|
| 802 |
+
</div>
|
| 803 |
+
</body>
|
| 804 |
+
</html>
|
| 805 |
+
"""
|
| 806 |
+
|
| 807 |
+
with open(output_path, "w", encoding="utf-8") as f:
|
| 808 |
+
f.write(html)
|
| 809 |
+
|
| 810 |
+
logger.info("تم تصدير مرجع HTML إلى: %s", output_path)
|
| 811 |
+
return str(output_path)
|
| 812 |
+
|
| 813 |
+
# ------------------------------------------------------------------
|
| 814 |
+
# أدوات مساعدة
|
| 815 |
+
# ------------------------------------------------------------------
|
| 816 |
+
|
| 817 |
+
@staticmethod
|
| 818 |
+
def _markdown_to_html(md_content: str) -> str:
|
| 819 |
+
"""تحويل Markdown بسيط إلى HTML.
|
| 820 |
+
|
| 821 |
+
يدعم: العناوين، الجداول، القوائم، الفواصل، الأكواد.
|
| 822 |
+
"""
|
| 823 |
+
if not md_content:
|
| 824 |
+
return ""
|
| 825 |
+
|
| 826 |
+
html_lines: list[str] = []
|
| 827 |
+
lines = md_content.split("\n")
|
| 828 |
+
in_table = False
|
| 829 |
+
in_code_block = False
|
| 830 |
+
|
| 831 |
+
for line in lines:
|
| 832 |
+
stripped = line.strip()
|
| 833 |
+
|
| 834 |
+
if not stripped:
|
| 835 |
+
continue
|
| 836 |
+
|
| 837 |
+
# كود
|
| 838 |
+
if stripped.startswith("```"):
|
| 839 |
+
if in_code_block:
|
| 840 |
+
html_lines.append("</code></pre>")
|
| 841 |
+
in_code_block = False
|
| 842 |
+
else:
|
| 843 |
+
lang = stripped[3:].strip()
|
| 844 |
+
html_lines.append(f"<pre><code class=\"language-{lang}\">" if lang else "<pre><code>")
|
| 845 |
+
in_code_block = True
|
| 846 |
+
continue
|
| 847 |
+
|
| 848 |
+
if in_code_block:
|
| 849 |
+
html_lines.append(stripped)
|
| 850 |
+
continue
|
| 851 |
+
|
| 852 |
+
# العناوين
|
| 853 |
+
if stripped.startswith("# "):
|
| 854 |
+
html_lines.append(f"<h1>{stripped[2:]}</h1>")
|
| 855 |
+
elif stripped.startswith("## "):
|
| 856 |
+
html_lines.append(f"<h2>{stripped[3:]}</h2>")
|
| 857 |
+
elif stripped.startswith("### "):
|
| 858 |
+
html_lines.append(f"<h3>{stripped[4:]}</h3>")
|
| 859 |
+
elif stripped.startswith("#### "):
|
| 860 |
+
html_lines.append(f"<h4>{stripped[5:]}</h4>")
|
| 861 |
+
|
| 862 |
+
# فاصل
|
| 863 |
+
elif stripped.startswith("---"):
|
| 864 |
+
if in_table:
|
| 865 |
+
html_lines.append("</table>")
|
| 866 |
+
in_table = False
|
| 867 |
+
html_lines.append("<hr>")
|
| 868 |
+
|
| 869 |
+
# جدول
|
| 870 |
+
elif stripped.startswith("|"):
|
| 871 |
+
cells = [c.strip() for c in stripped.split("|") if c.strip()]
|
| 872 |
+
|
| 873 |
+
# فاصل الجدول
|
| 874 |
+
if all(set(c) <= {"-", ":"} for c in cells):
|
| 875 |
+
continue
|
| 876 |
+
|
| 877 |
+
if not in_table:
|
| 878 |
+
html_lines.append("<table>")
|
| 879 |
+
# أول صف = رأس
|
| 880 |
+
header_cells = "".join(f"<th>{c}</th>" for c in cells)
|
| 881 |
+
html_lines.append(f"<tr>{header_cells}</tr>")
|
| 882 |
+
in_table = True
|
| 883 |
+
else:
|
| 884 |
+
row_cells = "".join(f"<td>{c}</td>" for c in cells)
|
| 885 |
+
html_lines.append(f"<tr>{row_cells}</tr>")
|
| 886 |
+
|
| 887 |
+
# قوائم
|
| 888 |
+
elif stripped.startswith("- ") or stripped.startswith("* "):
|
| 889 |
+
if in_table:
|
| 890 |
+
html_lines.append("</table>")
|
| 891 |
+
in_table = False
|
| 892 |
+
html_lines.append(f"<li>{stripped[2:]}</li>")
|
| 893 |
+
else:
|
| 894 |
+
# فحص قوائم مرقمة
|
| 895 |
+
import re
|
| 896 |
+
numbered_match = re.match(r"^(\d+)\.\s+(.+)$", stripped)
|
| 897 |
+
if numbered_match:
|
| 898 |
+
if in_table:
|
| 899 |
+
html_lines.append("</table>")
|
| 900 |
+
in_table = False
|
| 901 |
+
html_lines.append(f"<li>{numbered_match.group(2)}</li>")
|
| 902 |
+
else:
|
| 903 |
+
if in_table:
|
| 904 |
+
html_lines.append("</table>")
|
| 905 |
+
in_table = False
|
| 906 |
+
# فقرات عادية
|
| 907 |
+
html_lines.append(f"<p>{stripped}</p>")
|
| 908 |
+
|
| 909 |
+
if in_table:
|
| 910 |
+
html_lines.append("</table>")
|
| 911 |
+
if in_code_block:
|
| 912 |
+
html_lines.append("</code></pre>")
|
| 913 |
+
|
| 914 |
+
return "\n".join(html_lines)
|
| 915 |
+
|
| 916 |
+
@staticmethod
|
| 917 |
+
def _extract_vocabulary_from_items(
|
| 918 |
+
items: list[dict[str, Any]],
|
| 919 |
+
) -> list[dict[str, str]]:
|
| 920 |
+
"""استخراج أزواج المفردات (إنجليزي-عربي) من نتائج OCR.
|
| 921 |
+
|
| 922 |
+
Args:
|
| 923 |
+
items: قائمة نتائج OCR.
|
| 924 |
+
|
| 925 |
+
Returns:
|
| 926 |
+
قائمة أزواج {"english": ..., "arabic": ...}.
|
| 927 |
+
"""
|
| 928 |
+
vocab = []
|
| 929 |
+
|
| 930 |
+
for item in items:
|
| 931 |
+
text = str(item.get("text", "")).strip()
|
| 932 |
+
if not text:
|
| 933 |
+
continue
|
| 934 |
+
|
| 935 |
+
# فصل المكونات الإنجليزية والعربية
|
| 936 |
+
en_parts = []
|
| 937 |
+
ar_parts = []
|
| 938 |
+
|
| 939 |
+
words = text.split()
|
| 940 |
+
for word in words:
|
| 941 |
+
clean = word.strip(".,;:!?\"'()-*")
|
| 942 |
+
if not clean:
|
| 943 |
+
continue
|
| 944 |
+
|
| 945 |
+
arabic_chars = sum(1 for c in clean if "\u0600" <= c <= "\u06FF")
|
| 946 |
+
latin_chars = sum(1 for c in clean if c.isascii() and c.isalpha())
|
| 947 |
+
|
| 948 |
+
if arabic_chars > latin_chars:
|
| 949 |
+
ar_parts.append(clean)
|
| 950 |
+
elif latin_chars > 0:
|
| 951 |
+
en_parts.append(clean)
|
| 952 |
+
|
| 953 |
+
if en_parts or ar_parts:
|
| 954 |
+
vocab.append({
|
| 955 |
+
"english": " ".join(en_parts),
|
| 956 |
+
"arabic": " ".join(ar_parts),
|
| 957 |
+
})
|
| 958 |
+
|
| 959 |
+
return vocab
|
| 960 |
+
|
| 961 |
+
# ------------------------------------------------------------------
|
| 962 |
+
# توليد شامل
|
| 963 |
+
# ------------------------------------------------------------------
|
| 964 |
+
|
| 965 |
+
def generate_full_guide(
|
| 966 |
+
self,
|
| 967 |
+
ocr_results: list[dict[str, Any]],
|
| 968 |
+
output_dir: str | Path,
|
| 969 |
+
title: str = "مرجع دراسة شامل",
|
| 970 |
+
include_mermaid: bool = True,
|
| 971 |
+
include_flashcards: bool = True,
|
| 972 |
+
card_type: str = "bilingual",
|
| 973 |
+
max_cards: int = 200,
|
| 974 |
+
mermaid_type: str = "mindmap",
|
| 975 |
+
) -> dict[str, str]:
|
| 976 |
+
"""توليد مرجع دراسي شامل بكل المخرجات.
|
| 977 |
+
|
| 978 |
+
يُنشئ:
|
| 979 |
+
1. Markdown شامل
|
| 980 |
+
2. مخطط Mermaid
|
| 981 |
+
3. بطاقات Anki
|
| 982 |
+
4. HTML للطباعة
|
| 983 |
+
|
| 984 |
+
Args:
|
| 985 |
+
ocr_results: نتائج OCR.
|
| 986 |
+
output_dir: مجلد الحفظ.
|
| 987 |
+
title: عنوان المرجع.
|
| 988 |
+
include_mermaid: تضمين مخطط Mermaid.
|
| 989 |
+
include_flashcards: تضمين البطاقات.
|
| 990 |
+
card_type: نوع البطاقات.
|
| 991 |
+
max_cards: أقصى عدد بطاقات.
|
| 992 |
+
mermaid_type: نوع المخطط.
|
| 993 |
+
|
| 994 |
+
Returns:
|
| 995 |
+
قاموس بمسارات الملفات المُنشأة.
|
| 996 |
+
"""
|
| 997 |
+
output_dir = Path(output_dir)
|
| 998 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 999 |
+
|
| 1000 |
+
paths: dict[str, str] = {}
|
| 1001 |
+
|
| 1002 |
+
# 1. Markdown
|
| 1003 |
+
md_content = self.generate_markdown(ocr_results, title=title)
|
| 1004 |
+
if md_content:
|
| 1005 |
+
md_path = output_dir / "study_guide.md"
|
| 1006 |
+
with open(md_path, "w", encoding="utf-8") as f:
|
| 1007 |
+
f.write(md_content)
|
| 1008 |
+
paths["markdown"] = str(md_path)
|
| 1009 |
+
logger.info("تم حفظ المرجع Markdown: %s", md_path)
|
| 1010 |
+
|
| 1011 |
+
# استخراج المفردات
|
| 1012 |
+
vocab = self._extract_vocabulary_from_items(ocr_results)
|
| 1013 |
+
|
| 1014 |
+
# 2. Mermaid
|
| 1015 |
+
if include_mermaid and vocab:
|
| 1016 |
+
mermaid_code = self.generate_mermaid(
|
| 1017 |
+
vocab, diagram_type=mermaid_type, title=title,
|
| 1018 |
+
)
|
| 1019 |
+
if mermaid_code:
|
| 1020 |
+
mermaid_path = output_dir / "vocab_diagram.mmd"
|
| 1021 |
+
with open(mermaid_path, "w", encoding="utf-8") as f:
|
| 1022 |
+
f.write(mermaid_code)
|
| 1023 |
+
paths["mermaid"] = str(mermaid_path)
|
| 1024 |
+
|
| 1025 |
+
# تضمين Mermaid في Markdown
|
| 1026 |
+
md_content += "\n\n---\n\n## خريطة المفردات (Mermaid)\n\n"
|
| 1027 |
+
md_content += f"```mermaid\n{mermaid_code}\n```\n"
|
| 1028 |
+
|
| 1029 |
+
# إعادة حفظ Markdown المُحدَّث
|
| 1030 |
+
md_path = output_dir / "study_guide_full.md"
|
| 1031 |
+
with open(md_path, "w", encoding="utf-8") as f:
|
| 1032 |
+
f.write(md_content)
|
| 1033 |
+
paths["markdown_full"] = str(md_path)
|
| 1034 |
+
|
| 1035 |
+
# 3. Flashcards + Anki
|
| 1036 |
+
if include_flashcards and vocab:
|
| 1037 |
+
cards = self.generate_flashcards(
|
| 1038 |
+
vocab, card_type=card_type, max_cards=max_cards,
|
| 1039 |
+
)
|
| 1040 |
+
if cards:
|
| 1041 |
+
anki_path = output_dir / "flashcards_anki.csv"
|
| 1042 |
+
self.export_anki(cards, anki_path)
|
| 1043 |
+
paths["anki"] = str(anki_path)
|
| 1044 |
+
|
| 1045 |
+
# تضمين البطاقات في Markdown
|
| 1046 |
+
md_content += "\n\n---\n\n## البطاقات التعليمية\n\n"
|
| 1047 |
+
md_content += f"العدد: {len(cards)} بطاقة\n\n"
|
| 1048 |
+
for i, card in enumerate(cards[:50], 1):
|
| 1049 |
+
md_content += (
|
| 1050 |
+
f"**{i}.** الوجه: `{card['front']}`\n"
|
| 1051 |
+
f" الظهر: ||{card['back']}||\n\n"
|
| 1052 |
+
)
|
| 1053 |
+
|
| 1054 |
+
# إعادة حفظ
|
| 1055 |
+
md_path = output_dir / "study_guide_full.md"
|
| 1056 |
+
with open(md_path, "w", encoding="utf-8") as f:
|
| 1057 |
+
f.write(md_content)
|
| 1058 |
+
paths["markdown_full"] = str(md_path)
|
| 1059 |
+
|
| 1060 |
+
# 4. HTML
|
| 1061 |
+
html_path = output_dir / "study_guide.html"
|
| 1062 |
+
self.export_html(md_content, html_path, title=title)
|
| 1063 |
+
paths["html"] = str(html_path)
|
| 1064 |
+
|
| 1065 |
+
logger.info(
|
| 1066 |
+
"تم توليد مرجع شامل في: %s (%d ملف)",
|
| 1067 |
+
output_dir, len(paths),
|
| 1068 |
+
)
|
| 1069 |
+
|
| 1070 |
+
return paths
|
|
@@ -9,6 +9,8 @@
|
|
| 9 |
- المعالجة المسبقة (CLAHE, denoise, deskew)
|
| 10 |
- إعادة تجميع النصوص RTL
|
| 11 |
- دمج نتائج عدة محركات OCR
|
|
|
|
|
|
|
| 12 |
|
| 13 |
OmniFile AI Processor - وحدة معالجة الملفات الذكية
|
| 14 |
"""
|
|
@@ -19,6 +21,8 @@ from modules.vision.text_reconstructor import TextReconstructor
|
|
| 19 |
from modules.vision.result_fusion import ResultFusion, FusionStrategy
|
| 20 |
from modules.vision.layout_analyzer import LayoutAnalyzer
|
| 21 |
from modules.vision.table_extractor import TableExtractor
|
|
|
|
|
|
|
| 22 |
|
| 23 |
__all__ = [
|
| 24 |
"OCREngine",
|
|
@@ -29,4 +33,8 @@ __all__ = [
|
|
| 29 |
"FusionStrategy",
|
| 30 |
"LayoutAnalyzer",
|
| 31 |
"TableExtractor",
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
]
|
|
|
|
| 9 |
- المعالجة المسبقة (CLAHE, denoise, deskew)
|
| 10 |
- إعادة تجميع النصوص RTL
|
| 11 |
- دمج نتائج عدة محركات OCR
|
| 12 |
+
- استخراج النصوص من الفيديو (Video OCR)
|
| 13 |
+
- توسيع بيانات التدريب للكتابة اليدوية (Data Augmentation)
|
| 14 |
|
| 15 |
OmniFile AI Processor - وحدة معالجة الملفات الذكية
|
| 16 |
"""
|
|
|
|
| 21 |
from modules.vision.result_fusion import ResultFusion, FusionStrategy
|
| 22 |
from modules.vision.layout_analyzer import LayoutAnalyzer
|
| 23 |
from modules.vision.table_extractor import TableExtractor
|
| 24 |
+
from modules.vision.video_ocr import VideoOCR, FrameResult, VideoTimeline
|
| 25 |
+
from modules.vision.data_augmentation import DataAugmentor
|
| 26 |
|
| 27 |
__all__ = [
|
| 28 |
"OCREngine",
|
|
|
|
| 33 |
"FusionStrategy",
|
| 34 |
"LayoutAnalyzer",
|
| 35 |
"TableExtractor",
|
| 36 |
+
"VideoOCR",
|
| 37 |
+
"FrameResult",
|
| 38 |
+
"VideoTimeline",
|
| 39 |
+
"DataAugmentor",
|
| 40 |
]
|
|
@@ -0,0 +1,829 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
توسيع بيانات التدريب — Data Augmentation Module
|
| 3 |
+
===================================================
|
| 4 |
+
توليد صور مُعزَّزة لبيانات تدريب الكتابة اليدوية العربية.
|
| 5 |
+
|
| 6 |
+
القدرات:
|
| 7 |
+
- تقنيات متنوعة: دوران، ضوضاء، ضبابية، تباين/سطوع، تحويل منظوري، تشويه مرن
|
| 8 |
+
- مراعاة خصوصيات العربية: الحفاظ على التشكيل، معالجة RTL
|
| 9 |
+
- معالجة فردية وجماعية (batch)
|
| 10 |
+
- موازنة مجموعة البيانات بزيادة العينات النادرة
|
| 11 |
+
- دعم التسميات (labels) المرافقة
|
| 12 |
+
|
| 13 |
+
مثال الاستخدام:
|
| 14 |
+
>>> augmentor = DataAugmentor()
|
| 15 |
+
>>> # صورة واحدة
|
| 16 |
+
>>> augmented = augmentor.augment_image(image, num_augmented=5)
|
| 17 |
+
>>> # مجموعة بيانات
|
| 18 |
+
>>> augmented_dataset = augmentor.create_augmented_dataset(
|
| 19 |
+
... image_dir="data/train",
|
| 20 |
+
... label_file="data/labels.jsonl",
|
| 21 |
+
... output_dir="data/train_augmented",
|
| 22 |
+
... target_per_class=100,
|
| 23 |
+
... )
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
from __future__ import annotations
|
| 27 |
+
|
| 28 |
+
import json
|
| 29 |
+
import logging
|
| 30 |
+
import os
|
| 31 |
+
import random
|
| 32 |
+
import re
|
| 33 |
+
from collections import Counter
|
| 34 |
+
from pathlib import Path
|
| 35 |
+
from typing import Any, Optional, Union
|
| 36 |
+
|
| 37 |
+
logger = logging.getLogger(__name__)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# ======================================================================
|
| 41 |
+
# فئة مُعزِّز البيانات
|
| 42 |
+
# ======================================================================
|
| 43 |
+
|
| 44 |
+
class DataAugmentor:
|
| 45 |
+
"""مُعزِّز بيانات التدريب للكتابة اليدوية العربية.
|
| 46 |
+
|
| 47 |
+
توفر تقنيات متنوعة لتوسيع مجموعة بيانات التدريب مع مراعاة
|
| 48 |
+
خصوصيات النص العربي مثل التشكيل والحروف المتصلة.
|
| 49 |
+
|
| 50 |
+
Attributes:
|
| 51 |
+
seed: بذرة العشوائية (لإعادة النتائج).
|
| 52 |
+
techniques: قائمة التقنيات المُفعَّلة.
|
| 53 |
+
"""
|
| 54 |
+
|
| 55 |
+
# نطاق أحرف التشكيل العربية (Tashkeel / Diacritics)
|
| 56 |
+
ARABIC_DIACRITICS = set(
|
| 57 |
+
"\u0610\u0611\u0612\u0613\u0614\u0615\u0616"
|
| 58 |
+
"\u0617\u0618\u0619\u061A"
|
| 59 |
+
"\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652"
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
def __init__(
|
| 63 |
+
self,
|
| 64 |
+
seed: Optional[int] = 42,
|
| 65 |
+
techniques: Optional[list[str]] = None,
|
| 66 |
+
# إعدادات التقنيات
|
| 67 |
+
rotation_range: tuple[float, float] = (-5.0, 5.0),
|
| 68 |
+
noise_std_range: tuple[float, float] = (5.0, 25.0),
|
| 69 |
+
blur_kernel_range: tuple[int, int] = (1, 3),
|
| 70 |
+
brightness_range: tuple[float, float] = (0.8, 1.2),
|
| 71 |
+
contrast_range: tuple[float, float] = (0.8, 1.2),
|
| 72 |
+
perspective_scale: float = 0.02,
|
| 73 |
+
elastic_alpha: int = 30,
|
| 74 |
+
elastic_sigma: int = 4,
|
| 75 |
+
) -> None:
|
| 76 |
+
"""تهيئة مُعزِّز البيانات.
|
| 77 |
+
|
| 78 |
+
Args:
|
| 79 |
+
seed: بذرة العشوائية (None = عشوائي).
|
| 80 |
+
techniques: قائمة التقنيات المُفعَّلة.
|
| 81 |
+
الخيارات: "rotation", "noise", "blur", "brightness",
|
| 82 |
+
"contrast", "perspective", "elastic".
|
| 83 |
+
None = كل التقنيات.
|
| 84 |
+
rotation_range: نطاق زاوية الدوران بالدرجات.
|
| 85 |
+
noise_std_range: نطاق الانحراف المعياري للضوضاء.
|
| 86 |
+
blur_kernel_range: نطاق حجم نواة الضبابية.
|
| 87 |
+
brightness_range: نطاق عامل السطوع.
|
| 88 |
+
contrast_range: نطاق عامل التباين.
|
| 89 |
+
perspective_scale: مقياس تحويل المنظور.
|
| 90 |
+
elastic_alpha: شدة التشويه المرن (elastic).
|
| 91 |
+
elastic_sigma: انتشار التشويه المرن.
|
| 92 |
+
"""
|
| 93 |
+
if seed is not None:
|
| 94 |
+
random.seed(seed)
|
| 95 |
+
|
| 96 |
+
self.seed = seed
|
| 97 |
+
self.rotation_range = rotation_range
|
| 98 |
+
self.noise_std_range = noise_std_range
|
| 99 |
+
self.blur_kernel_range = blur_kernel_range
|
| 100 |
+
self.brightness_range = brightness_range
|
| 101 |
+
self.contrast_range = contrast_range
|
| 102 |
+
self.perspective_scale = perspective_scale
|
| 103 |
+
self.elastic_alpha = elastic_alpha
|
| 104 |
+
self.elastic_sigma = elastic_sigma
|
| 105 |
+
|
| 106 |
+
# التقنيات المتاحة
|
| 107 |
+
self._all_techniques = {
|
| 108 |
+
"rotation",
|
| 109 |
+
"noise",
|
| 110 |
+
"blur",
|
| 111 |
+
"brightness",
|
| 112 |
+
"contrast",
|
| 113 |
+
"perspective",
|
| 114 |
+
"elastic",
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
self.techniques = set(techniques) if techniques else self._all_techniques
|
| 118 |
+
|
| 119 |
+
# التحقق من التقنيات
|
| 120 |
+
invalid = self.techniques - self._all_techniques
|
| 121 |
+
if invalid:
|
| 122 |
+
logger.warning("تقنيات غير معروفة (تم التجاهل): %s", invalid)
|
| 123 |
+
self.techniques -= invalid
|
| 124 |
+
|
| 125 |
+
# التحقق من المكتبات
|
| 126 |
+
self._has_cv2 = self._check_library("cv2", "opencv-python")
|
| 127 |
+
self._has_pil = self._check_library("PIL", "Pillow")
|
| 128 |
+
self._has_numpy = self._check_library("numpy", "numpy")
|
| 129 |
+
|
| 130 |
+
if not self._has_numpy:
|
| 131 |
+
logger.error("NumPy غير متاح — مطلوب لجميع عمليات التعزيز")
|
| 132 |
+
|
| 133 |
+
@staticmethod
|
| 134 |
+
def _check_library(import_name: str, package_name: str) -> bool:
|
| 135 |
+
"""التحقق من توفر مكتبة."""
|
| 136 |
+
try:
|
| 137 |
+
__import__(import_name)
|
| 138 |
+
return True
|
| 139 |
+
except ImportError:
|
| 140 |
+
return False
|
| 141 |
+
|
| 142 |
+
# ------------------------------------------------------------------
|
| 143 |
+
# تحويل الصور
|
| 144 |
+
# ------------------------------------------------------------------
|
| 145 |
+
|
| 146 |
+
def _to_numpy(self, image: Any) -> Any:
|
| 147 |
+
"""تحويل الصورة إلى مصفوفة numpy."""
|
| 148 |
+
if self._has_numpy:
|
| 149 |
+
import numpy as np
|
| 150 |
+
if isinstance(image, np.ndarray):
|
| 151 |
+
return image.copy()
|
| 152 |
+
try:
|
| 153 |
+
from PIL import Image
|
| 154 |
+
if isinstance(image, Image.Image):
|
| 155 |
+
import numpy as np
|
| 156 |
+
return np.array(image)
|
| 157 |
+
except (ImportError, Exception):
|
| 158 |
+
pass
|
| 159 |
+
raise TypeError(f"نوع غير مدعوم: {type(image)} — مطلوب PIL.Image أو numpy.ndarray")
|
| 160 |
+
|
| 161 |
+
def _to_pil(self, img_array: Any) -> Any:
|
| 162 |
+
"""تحويل مصفوفة numpy إلى صورة PIL."""
|
| 163 |
+
try:
|
| 164 |
+
from PIL import Image
|
| 165 |
+
except ImportError:
|
| 166 |
+
raise RuntimeError("Pillow غير متاح")
|
| 167 |
+
|
| 168 |
+
import numpy as np
|
| 169 |
+
|
| 170 |
+
if img_array.ndim == 2:
|
| 171 |
+
return Image.fromarray(img_array, mode="L").convert("RGB")
|
| 172 |
+
elif img_array.ndim == 3:
|
| 173 |
+
if img_array.shape[2] == 3:
|
| 174 |
+
return Image.fromarray(img_array, mode="RGB")
|
| 175 |
+
elif img_array.shape[2] == 4:
|
| 176 |
+
return Image.fromarray(img_array, mode="RGBA")
|
| 177 |
+
return Image.fromarray(img_array).convert("RGB")
|
| 178 |
+
|
| 179 |
+
# ------------------------------------------------------------------
|
| 180 |
+
# تقنيات التعزيز الفردية
|
| 181 |
+
# ------------------------------------------------------------------
|
| 182 |
+
|
| 183 |
+
def _apply_rotation(self, img: Any) -> Any:
|
| 184 |
+
"""تطبيق دوران عشوائي على الصورة.
|
| 185 |
+
|
| 186 |
+
Args:
|
| 187 |
+
img: مصفوفة numpy.
|
| 188 |
+
|
| 189 |
+
Returns:
|
| 190 |
+
الصورة بعد الدوران.
|
| 191 |
+
"""
|
| 192 |
+
if not self._has_cv2:
|
| 193 |
+
return img
|
| 194 |
+
|
| 195 |
+
import cv2
|
| 196 |
+
import numpy as np
|
| 197 |
+
|
| 198 |
+
h, w = img.shape[:2]
|
| 199 |
+
angle = random.uniform(*self.rotation_range)
|
| 200 |
+
center = (w / 2, h / 2)
|
| 201 |
+
|
| 202 |
+
rotation_matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
|
| 203 |
+
|
| 204 |
+
# حساب الحدود الجديدة لتجنب القص
|
| 205 |
+
cos_val = abs(rotation_matrix[0, 0])
|
| 206 |
+
sin_val = abs(rotation_matrix[0, 1])
|
| 207 |
+
new_w = int(h * sin_val + w * cos_val)
|
| 208 |
+
new_h = int(h * cos_val + w * sin_val)
|
| 209 |
+
|
| 210 |
+
rotation_matrix[0, 2] += (new_w - w) / 2
|
| 211 |
+
rotation_matrix[1, 2] += (new_h - h) / 2
|
| 212 |
+
|
| 213 |
+
return cv2.warpAffine(
|
| 214 |
+
img, rotation_matrix, (new_w, new_h),
|
| 215 |
+
flags=cv2.INTER_CUBIC,
|
| 216 |
+
borderMode=cv2.BORDER_REPLICATE,
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
def _apply_noise(self, img: Any) -> Any:
|
| 220 |
+
"""إضافة ضوضاء غاوسية عشوائية.
|
| 221 |
+
|
| 222 |
+
Args:
|
| 223 |
+
img: مصفوفة numpy.
|
| 224 |
+
|
| 225 |
+
Returns:
|
| 226 |
+
الصورة مع الضوضاء.
|
| 227 |
+
"""
|
| 228 |
+
import numpy as np
|
| 229 |
+
|
| 230 |
+
std = random.uniform(*self.noise_std_range)
|
| 231 |
+
noise = np.random.normal(0, std, img.shape).astype(np.float32)
|
| 232 |
+
noisy = np.clip(img.astype(np.float32) + noise, 0, 255)
|
| 233 |
+
|
| 234 |
+
return noisy.astype(img.dtype)
|
| 235 |
+
|
| 236 |
+
def _apply_blur(self, img: Any) -> Any:
|
| 237 |
+
"""تطبيق ضبابية غاوسية عشوائية.
|
| 238 |
+
|
| 239 |
+
Args:
|
| 240 |
+
img: مصفوفة numpy.
|
| 241 |
+
|
| 242 |
+
Returns:
|
| 243 |
+
الصورة بعد الضبابية.
|
| 244 |
+
"""
|
| 245 |
+
if not self._has_cv2:
|
| 246 |
+
return img
|
| 247 |
+
|
| 248 |
+
import cv2
|
| 249 |
+
|
| 250 |
+
kernel_size = random.choice(
|
| 251 |
+
list(range(self.blur_kernel_range[0], self.blur_kernel_range[1] + 1, 2))
|
| 252 |
+
)
|
| 253 |
+
if kernel_size <= 1:
|
| 254 |
+
return img
|
| 255 |
+
|
| 256 |
+
return cv2.GaussianBlur(img, (kernel_size, kernel_size), 0)
|
| 257 |
+
|
| 258 |
+
def _apply_brightness_contrast(self, img: Any) -> Any:
|
| 259 |
+
"""تعديل السطوع والتباين عشوائياً.
|
| 260 |
+
|
| 261 |
+
Args:
|
| 262 |
+
img: مصفوفة numpy.
|
| 263 |
+
|
| 264 |
+
Returns:
|
| 265 |
+
الصورة بعد التعديل.
|
| 266 |
+
"""
|
| 267 |
+
import numpy as np
|
| 268 |
+
|
| 269 |
+
brightness = random.uniform(*self.brightness_range)
|
| 270 |
+
contrast = random.uniform(*self.contrast_range)
|
| 271 |
+
|
| 272 |
+
result = img.astype(np.float32) * contrast * brightness
|
| 273 |
+
return np.clip(result, 0, 255).astype(img.dtype)
|
| 274 |
+
|
| 275 |
+
def _apply_perspective(self, img: Any) -> Any:
|
| 276 |
+
"""تطبيق تحويل منظوري خفيف.
|
| 277 |
+
|
| 278 |
+
مفيد لمحاكاة زوايا التصوير المختلفة للخط اليدوي.
|
| 279 |
+
|
| 280 |
+
Args:
|
| 281 |
+
img: مصفوفة numpy.
|
| 282 |
+
|
| 283 |
+
Returns:
|
| 284 |
+
الصورة بعد التحويل.
|
| 285 |
+
"""
|
| 286 |
+
if not self._has_cv2:
|
| 287 |
+
return img
|
| 288 |
+
|
| 289 |
+
import cv2
|
| 290 |
+
import numpy as np
|
| 291 |
+
|
| 292 |
+
h, w = img.shape[:2]
|
| 293 |
+
scale = self.perspective_scale
|
| 294 |
+
|
| 295 |
+
# نقاط المصدر
|
| 296 |
+
src_points = np.float32([
|
| 297 |
+
[0, 0], [w, 0], [w, h], [0, h],
|
| 298 |
+
])
|
| 299 |
+
|
| 300 |
+
# نقاط الهدف مع تشويه عشوائي
|
| 301 |
+
dx = random.uniform(-scale, scale) * w
|
| 302 |
+
dy = random.uniform(-scale, scale) * h
|
| 303 |
+
dst_points = np.float32([
|
| 304 |
+
[dx, dy],
|
| 305 |
+
[w - dx, dy + random.uniform(-scale, scale) * h * 0.5],
|
| 306 |
+
[w - dx - random.uniform(-scale, scale) * w * 0.5, h - dy],
|
| 307 |
+
[dx + random.uniform(-scale, scale) * w * 0.5, h - dy],
|
| 308 |
+
])
|
| 309 |
+
|
| 310 |
+
matrix = cv2.getPerspectiveTransform(src_points, dst_points)
|
| 311 |
+
return cv2.warpPerspective(img, matrix, (w, h), flags=cv2.INTER_CUBIC)
|
| 312 |
+
|
| 313 |
+
def _apply_elastic(self, img: Any) -> Any:
|
| 314 |
+
"""تطبيق تشويه مرن (elastic distortion).
|
| 315 |
+
|
| 316 |
+
يُحاكي التشوهات الطبيعية في الكتابة اليدوية.
|
| 317 |
+
|
| 318 |
+
Args:
|
| 319 |
+
img: مصفوفة numpy.
|
| 320 |
+
|
| 321 |
+
Returns:
|
| 322 |
+
الصورة بعد التشويه.
|
| 323 |
+
"""
|
| 324 |
+
import numpy as np
|
| 325 |
+
|
| 326 |
+
if not self._has_cv2:
|
| 327 |
+
# احتياطي بسيط بدون OpenCV
|
| 328 |
+
alpha = self.elastic_alpha / 255.0
|
| 329 |
+
sigma = self.elastic_sigma
|
| 330 |
+
|
| 331 |
+
h, w = img.shape[:2]
|
| 332 |
+
dx = np.random.uniform(-1, 1, (h, w)).astype(np.float32) * alpha
|
| 333 |
+
dy = np.random.uniform(-1, 1, (h, w)).astype(np.float32) * alpha
|
| 334 |
+
|
| 335 |
+
# تطبيق Gaussian blur يدوياً على الإزاحة
|
| 336 |
+
from PIL import ImageFilter
|
| 337 |
+
dx_img = Image.fromarray((dx * 255).astype(np.uint8))
|
| 338 |
+
dy_img = Image.fromarray((dy * 255).astype(np.uint8))
|
| 339 |
+
dx_img = dx_img.filter(ImageFilter.GaussianBlur(sigma))
|
| 340 |
+
dy_img = dy_img.filter(ImageFilter.GaussianBlur(sigma))
|
| 341 |
+
dx = np.array(dx_img).astype(np.float32) / 255.0 * alpha
|
| 342 |
+
dy = np.array(dy_img).astype(np.float32) / 255.0 * alpha
|
| 343 |
+
|
| 344 |
+
# تطبيق الإزاحة
|
| 345 |
+
y_indices, x_indices = np.mgrid[:h, :w]
|
| 346 |
+
new_x = np.clip(x_indices + dx, 0, w - 1).astype(int)
|
| 347 |
+
new_y = np.clip(y_indices + dy, 0, h - 1).astype(int)
|
| 348 |
+
|
| 349 |
+
if img.ndim == 3:
|
| 350 |
+
return img[new_y, new_x, :]
|
| 351 |
+
return img[new_y, new_x]
|
| 352 |
+
|
| 353 |
+
import cv2
|
| 354 |
+
|
| 355 |
+
alpha = self.elastic_alpha
|
| 356 |
+
sigma = self.elastic_sigma
|
| 357 |
+
|
| 358 |
+
shape = img.shape[:2]
|
| 359 |
+
if img.ndim == 3:
|
| 360 |
+
shape = img.shape[:2]
|
| 361 |
+
|
| 362 |
+
# إنشاء حقول الإزاحة العشوائية
|
| 363 |
+
dx = np.random.uniform(-1, 1, shape).astype(np.float32) * alpha
|
| 364 |
+
dy = np.random.uniform(-1, 1, shape).astype(np.float32) * alpha
|
| 365 |
+
|
| 366 |
+
# تنعيم بحساب Gaussian blur
|
| 367 |
+
dx = cv2.GaussianBlur(dx, (0, 0), sigma)
|
| 368 |
+
dy = cv2.GaussianBlur(dy, (0, 0), sigma)
|
| 369 |
+
|
| 370 |
+
# إنشاء خريطة الإزاحة
|
| 371 |
+
h, w = shape
|
| 372 |
+
x, y = np.meshgrid(np.arange(w), np.arange(h))
|
| 373 |
+
map_x = (x + dx).astype(np.float32)
|
| 374 |
+
map_y = (y + dy).astype(np.float32)
|
| 375 |
+
|
| 376 |
+
return cv2.remap(img, map_x, map_y, cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE)
|
| 377 |
+
|
| 378 |
+
# ------------------------------------------------------------------
|
| 379 |
+
# الواجهة العامة
|
| 380 |
+
# ------------------------------------------------------------------
|
| 381 |
+
|
| 382 |
+
def augment_image(
|
| 383 |
+
self,
|
| 384 |
+
image: Any,
|
| 385 |
+
num_augmented: int = 5,
|
| 386 |
+
techniques: Optional[list[str]] = None,
|
| 387 |
+
return_numpy: bool = False,
|
| 388 |
+
) -> list[Any]:
|
| 389 |
+
"""تعزيز صورة واحدة بعدة تقنيات عشوائية.
|
| 390 |
+
|
| 391 |
+
Args:
|
| 392 |
+
image: صورة PIL أو مصفوفة numpy.
|
| 393 |
+
num_augmented: عدد النسخ المُعزَّزة المطلوبة.
|
| 394 |
+
techniques: تقنيات مُحددة (None = كل التقنيات المُفعَّلة).
|
| 395 |
+
return_numpy: إرجاع numpy بدلاً من PIL.
|
| 396 |
+
|
| 397 |
+
Returns:
|
| 398 |
+
قائمة بالصور المُعزَّزة.
|
| 399 |
+
|
| 400 |
+
Example:
|
| 401 |
+
>>> augmented = augmentor.augment_image(img, num_augmented=5)
|
| 402 |
+
"""
|
| 403 |
+
if num_augmented <= 0:
|
| 404 |
+
return []
|
| 405 |
+
|
| 406 |
+
import numpy as np
|
| 407 |
+
|
| 408 |
+
img_array = self._to_numpy(image)
|
| 409 |
+
active_techniques = set(techniques) if techniques else self.techniques
|
| 410 |
+
|
| 411 |
+
if not active_techniques:
|
| 412 |
+
logger.warning("لا توجد تقنيات مُفعَّلة للتعزيز")
|
| 413 |
+
return []
|
| 414 |
+
|
| 415 |
+
augmented: list[Any] = []
|
| 416 |
+
technique_list = [
|
| 417 |
+
"rotation", "noise", "blur",
|
| 418 |
+
"brightness", "contrast", "perspective", "elastic",
|
| 419 |
+
]
|
| 420 |
+
# الفلترة حسب التقنيات الفعالة
|
| 421 |
+
technique_list = [t for t in technique_list if t in active_techniques]
|
| 422 |
+
|
| 423 |
+
if not technique_list:
|
| 424 |
+
return []
|
| 425 |
+
|
| 426 |
+
for _ in range(num_augmented):
|
| 427 |
+
current = img_array.copy()
|
| 428 |
+
|
| 429 |
+
# اختيار 1-3 تقنيات عشوائية
|
| 430 |
+
num_techniques = random.randint(1, min(3, len(technique_list)))
|
| 431 |
+
chosen = random.sample(technique_list, num_techniques)
|
| 432 |
+
|
| 433 |
+
for technique in chosen:
|
| 434 |
+
try:
|
| 435 |
+
if technique == "rotation":
|
| 436 |
+
current = self._apply_rotation(current)
|
| 437 |
+
elif technique == "noise":
|
| 438 |
+
current = self._apply_noise(current)
|
| 439 |
+
elif technique == "blur":
|
| 440 |
+
current = self._apply_blur(current)
|
| 441 |
+
elif technique in ("brightness", "contrast"):
|
| 442 |
+
current = self._apply_brightness_contrast(current)
|
| 443 |
+
elif technique == "perspective":
|
| 444 |
+
current = self._apply_perspective(current)
|
| 445 |
+
elif technique == "elastic":
|
| 446 |
+
current = self._apply_elastic(current)
|
| 447 |
+
except Exception as e:
|
| 448 |
+
logger.debug("فشل تطبيق تقنية '%s': %s", technique, e)
|
| 449 |
+
|
| 450 |
+
if return_numpy:
|
| 451 |
+
augmented.append(current)
|
| 452 |
+
else:
|
| 453 |
+
augmented.append(self._to_pil(current))
|
| 454 |
+
|
| 455 |
+
logger.debug(
|
| 456 |
+
"تم تعزيز صورة واحدة: %d نسخة بـ %d تقنية",
|
| 457 |
+
num_augmented, len(chosen),
|
| 458 |
+
)
|
| 459 |
+
return augmented
|
| 460 |
+
|
| 461 |
+
def augment_with_label(
|
| 462 |
+
self,
|
| 463 |
+
image: Any,
|
| 464 |
+
label: str,
|
| 465 |
+
num_augmented: int = 5,
|
| 466 |
+
return_numpy: bool = False,
|
| 467 |
+
) -> list[dict[str, Any]]:
|
| 468 |
+
"""تعزيز صورة مع الحفاظ على التسمية المرافقة.
|
| 469 |
+
|
| 470 |
+
للنصوص العربية: يتحقق من أن التعزيز لا يؤثر على التشكيل.
|
| 471 |
+
|
| 472 |
+
Args:
|
| 473 |
+
image: صورة PIL أو numpy.
|
| 474 |
+
label: النص المرافق (التسمية).
|
| 475 |
+
num_augmented: عدد النسخ.
|
| 476 |
+
return_numpy: إرجاع numpy.
|
| 477 |
+
|
| 478 |
+
Returns:
|
| 479 |
+
قائمة قواميس: {"image": ..., "label": ...}.
|
| 480 |
+
"""
|
| 481 |
+
if not label.strip():
|
| 482 |
+
logger.warning("تسمية فارغة — تم التجاهل")
|
| 483 |
+
return []
|
| 484 |
+
|
| 485 |
+
augmented_images = self.augment_image(
|
| 486 |
+
image, num_augmented=num_augmented, return_numpy=return_numpy,
|
| 487 |
+
)
|
| 488 |
+
|
| 489 |
+
results = []
|
| 490 |
+
for aug_img in augmented_images:
|
| 491 |
+
# للنصوص العربية: التسمية تبقى كما هي لأن التعزيز بصري فقط
|
| 492 |
+
results.append({
|
| 493 |
+
"image": aug_img,
|
| 494 |
+
"label": label,
|
| 495 |
+
})
|
| 496 |
+
|
| 497 |
+
return results
|
| 498 |
+
|
| 499 |
+
def augment_dataset(
|
| 500 |
+
self,
|
| 501 |
+
images: list[Any],
|
| 502 |
+
labels: Optional[list[str]] = None,
|
| 503 |
+
num_augmented: int = 3,
|
| 504 |
+
return_numpy: bool = False,
|
| 505 |
+
) -> list[dict[str, Any]]:
|
| 506 |
+
"""تعزيز مجموعة صور كاملة.
|
| 507 |
+
|
| 508 |
+
Args:
|
| 509 |
+
images: قائمة صور (PIL أو numpy).
|
| 510 |
+
labels: قائمة تسميات مرافقة (اختياري).
|
| 511 |
+
num_augmented: عدد النسخ لكل صورة.
|
| 512 |
+
return_numpy: إرجاع numpy.
|
| 513 |
+
|
| 514 |
+
Returns:
|
| 515 |
+
قائمة نتائج: [{"image": ..., "label": ...}, ...].
|
| 516 |
+
|
| 517 |
+
Example:
|
| 518 |
+
>>> results = augmentor.augment_dataset(
|
| 519 |
+
... images=[img1, img2, img3],
|
| 520 |
+
... labels=["مرحبا", "عالم", "برمجة"],
|
| 521 |
+
... num_augmented=5,
|
| 522 |
+
... )
|
| 523 |
+
"""
|
| 524 |
+
if not images:
|
| 525 |
+
return []
|
| 526 |
+
|
| 527 |
+
results: list[dict[str, Any]] = []
|
| 528 |
+
total = len(images)
|
| 529 |
+
|
| 530 |
+
for idx, img in enumerate(images):
|
| 531 |
+
label = labels[idx] if labels and idx < len(labels) else ""
|
| 532 |
+
|
| 533 |
+
if label:
|
| 534 |
+
augmented = self.augment_with_label(
|
| 535 |
+
img, label, num_augmented=num_augmented, return_numpy=return_numpy,
|
| 536 |
+
)
|
| 537 |
+
else:
|
| 538 |
+
aug_images = self.augment_image(
|
| 539 |
+
img, num_augmented=num_augmented, return_numpy=return_numpy,
|
| 540 |
+
)
|
| 541 |
+
augmented = [{"image": a, "label": ""} for a in aug_images]
|
| 542 |
+
|
| 543 |
+
results.extend(augmented)
|
| 544 |
+
|
| 545 |
+
if (idx + 1) % 100 == 0 or idx == total - 1:
|
| 546 |
+
logger.info(
|
| 547 |
+
"تعزيز مجموعة البيانات: %d/%d صورة (%d عيّنة)",
|
| 548 |
+
idx + 1, total, len(results),
|
| 549 |
+
)
|
| 550 |
+
|
| 551 |
+
logger.info(
|
| 552 |
+
"تم تعزيز مجموعة البيانات: %d صورة أصلية → %d عيّنة",
|
| 553 |
+
total, len(results),
|
| 554 |
+
)
|
| 555 |
+
return results
|
| 556 |
+
|
| 557 |
+
def create_augmented_dataset(
|
| 558 |
+
self,
|
| 559 |
+
image_dir: str | Path,
|
| 560 |
+
label_file: Optional[str | Path] = None,
|
| 561 |
+
output_dir: Optional[str | Path] = None,
|
| 562 |
+
num_augmented: int = 5,
|
| 563 |
+
target_per_class: Optional[int] = None,
|
| 564 |
+
return_numpy: bool = False,
|
| 565 |
+
) -> dict[str, Any]:
|
| 566 |
+
"""إنشاء مجموعة بيانات مُعزَّزة من مجلد صور مع تسميات.
|
| 567 |
+
|
| 568 |
+
يدعم:
|
| 569 |
+
- قراءة الصور من مجلد
|
| 570 |
+
- قراءة التسميات من ملف JSONL أو JSON
|
| 571 |
+
- موازنة الفئات بزيادة العينات النادرة
|
| 572 |
+
- حفظ النتائج في مجلد مُخرَج
|
| 573 |
+
|
| 574 |
+
Args:
|
| 575 |
+
image_dir: مجلد الصور الأصلية.
|
| 576 |
+
label_file: ملف التسميات (JSONL أو JSON).
|
| 577 |
+
صيغة JSONL: سطر لكل صورة {"image": "path", "label": "text"}
|
| 578 |
+
صيغة JSON: {"image_name": "label", ...}
|
| 579 |
+
output_dir: مجلد حفظ النتائج (None = لا يحفظ).
|
| 580 |
+
num_augmented: عدد النسخ لكل صورة.
|
| 581 |
+
target_per_class: هدف عدد العينات لكل فئة (لموازنة البيانات).
|
| 582 |
+
return_numpy: إرجاع numpy.
|
| 583 |
+
|
| 584 |
+
Returns:
|
| 585 |
+
قاموس يحتوي:
|
| 586 |
+
- total_original: عدد الصور الأصلية
|
| 587 |
+
- total_augmented: عدد النسخ المُعزَّزة
|
| 588 |
+
- class_distribution: توزيع الفئات قبل وبعد
|
| 589 |
+
- results: قائمة النتائج
|
| 590 |
+
"""
|
| 591 |
+
image_dir = Path(image_dir)
|
| 592 |
+
if not image_dir.exists():
|
| 593 |
+
raise FileNotFoundError(f"مجلد الصور غير موجود: {image_dir}")
|
| 594 |
+
|
| 595 |
+
# تحميل التسميات
|
| 596 |
+
labels_map: dict[str, str] = {}
|
| 597 |
+
if label_file:
|
| 598 |
+
label_file = Path(label_file)
|
| 599 |
+
if not label_file.exists():
|
| 600 |
+
logger.warning("ملف التسميات غير موجود: %s", label_file)
|
| 601 |
+
else:
|
| 602 |
+
labels_map = self._load_labels(label_file)
|
| 603 |
+
|
| 604 |
+
# تحميل الصور
|
| 605 |
+
image_extensions = {".png", ".jpg", ".jpeg", ".bmp", ".tiff", ".tif", ".webp"}
|
| 606 |
+
image_files = sorted([
|
| 607 |
+
f for f in image_dir.iterdir()
|
| 608 |
+
if f.suffix.lower() in image_extensions
|
| 609 |
+
])
|
| 610 |
+
|
| 611 |
+
if not image_files:
|
| 612 |
+
logger.warning("لا توجد صور في المجلد: %s", image_dir)
|
| 613 |
+
return {
|
| 614 |
+
"total_original": 0,
|
| 615 |
+
"total_augmented": 0,
|
| 616 |
+
"class_distribution": {"before": {}, "after": {}},
|
| 617 |
+
"results": [],
|
| 618 |
+
}
|
| 619 |
+
|
| 620 |
+
# حساب التوزيع قبل التعزيز
|
| 621 |
+
original_labels = [labels_map.get(f.name, "") for f in image_files]
|
| 622 |
+
before_dist = dict(Counter(original_labels))
|
| 623 |
+
|
| 624 |
+
# حساب عدد النسخ لكل فئة
|
| 625 |
+
augmented_per_image: dict[str, int] = {}
|
| 626 |
+
if target_per_class is not None:
|
| 627 |
+
for label, count in before_dist.items():
|
| 628 |
+
if count < target_per_class and label:
|
| 629 |
+
needed = target_per_class - count
|
| 630 |
+
copies_per_image = max(1, needed // count) if count > 0 else needed
|
| 631 |
+
augmented_per_image[label] = copies_per_image
|
| 632 |
+
elif label:
|
| 633 |
+
augmented_per_image[label] = num_augmented
|
| 634 |
+
|
| 635 |
+
# التعزيز
|
| 636 |
+
all_results: list[dict[str, Any]] = []
|
| 637 |
+
for img_path in image_files:
|
| 638 |
+
try:
|
| 639 |
+
from PIL import Image
|
| 640 |
+
img = Image.open(img_path).convert("RGB")
|
| 641 |
+
label = labels_map.get(img_path.name, "")
|
| 642 |
+
|
| 643 |
+
# تحديد عدد النسخ
|
| 644 |
+
if label and label in augmented_per_image:
|
| 645 |
+
n_copies = augmented_per_image[label]
|
| 646 |
+
else:
|
| 647 |
+
n_copies = num_augmented
|
| 648 |
+
|
| 649 |
+
augmented = self.augment_with_label(
|
| 650 |
+
img, label, num_augmented=n_copies, return_numpy=return_numpy,
|
| 651 |
+
)
|
| 652 |
+
|
| 653 |
+
for aug in augmented:
|
| 654 |
+
aug["source_image"] = img_path.name
|
| 655 |
+
|
| 656 |
+
all_results.extend(augmented)
|
| 657 |
+
|
| 658 |
+
except Exception as e:
|
| 659 |
+
logger.warning("فشل في معالجة الصورة %s: %s", img_path.name, e)
|
| 660 |
+
|
| 661 |
+
# حساب التوزيع بعد التعزيز
|
| 662 |
+
after_labels = [r["label"] for r in all_results if r["label"]]
|
| 663 |
+
after_dist = dict(Counter(after_labels))
|
| 664 |
+
|
| 665 |
+
# حفظ النتائج
|
| 666 |
+
if output_dir:
|
| 667 |
+
output_dir = Path(output_dir)
|
| 668 |
+
self._save_augmented_dataset(all_results, output_dir)
|
| 669 |
+
|
| 670 |
+
logger.info(
|
| 671 |
+
"تم إنشاء مجموعة بيانات مُعزَّزة: %d أصلية → %d عيّنة",
|
| 672 |
+
len(image_files), len(all_results),
|
| 673 |
+
)
|
| 674 |
+
|
| 675 |
+
return {
|
| 676 |
+
"total_original": len(image_files),
|
| 677 |
+
"total_augmented": len(all_results),
|
| 678 |
+
"class_distribution": {
|
| 679 |
+
"before": before_dist,
|
| 680 |
+
"after": after_dist,
|
| 681 |
+
},
|
| 682 |
+
"results": all_results,
|
| 683 |
+
}
|
| 684 |
+
|
| 685 |
+
# ------------------------------------------------------------------
|
| 686 |
+
# أدوات مساعدة
|
| 687 |
+
# ------------------------------------------------------------------
|
| 688 |
+
|
| 689 |
+
@staticmethod
|
| 690 |
+
def _load_labels(label_file: Path) -> dict[str, str]:
|
| 691 |
+
"""تحميل التسميات من ملف JSONL أو JSON.
|
| 692 |
+
|
| 693 |
+
Args:
|
| 694 |
+
label_file: مسار ملف التسميات.
|
| 695 |
+
|
| 696 |
+
Returns:
|
| 697 |
+
قاموس {اسم_الصورة: التسمية}.
|
| 698 |
+
"""
|
| 699 |
+
labels: dict[str, str] = {}
|
| 700 |
+
|
| 701 |
+
try:
|
| 702 |
+
with open(label_file, "r", encoding="utf-8") as f:
|
| 703 |
+
if label_file.suffix.lower() == ".jsonl":
|
| 704 |
+
for line in f:
|
| 705 |
+
line = line.strip()
|
| 706 |
+
if not line:
|
| 707 |
+
continue
|
| 708 |
+
entry = json.loads(line)
|
| 709 |
+
img_name = entry.get("image", entry.get("file", entry.get("filename", "")))
|
| 710 |
+
label = entry.get("label", entry.get("text", ""))
|
| 711 |
+
if img_name and label:
|
| 712 |
+
labels[Path(img_name).name] = str(label)
|
| 713 |
+
else:
|
| 714 |
+
data = json.load(f)
|
| 715 |
+
if isinstance(data, dict):
|
| 716 |
+
for key, value in data.items():
|
| 717 |
+
labels[Path(key).name] = str(value)
|
| 718 |
+
elif isinstance(data, list):
|
| 719 |
+
for entry in data:
|
| 720 |
+
if isinstance(entry, dict):
|
| 721 |
+
img_name = entry.get("image", entry.get("file", ""))
|
| 722 |
+
label = entry.get("label", entry.get("text", ""))
|
| 723 |
+
if img_name and label:
|
| 724 |
+
labels[Path(img_name).name] = str(label)
|
| 725 |
+
|
| 726 |
+
logger.info("تم تحميل %d تسمية من %s", len(labels), label_file.name)
|
| 727 |
+
except Exception as e:
|
| 728 |
+
logger.error("فشل في تحميل التسميات: %s", e)
|
| 729 |
+
|
| 730 |
+
return labels
|
| 731 |
+
|
| 732 |
+
def _save_augmented_dataset(
|
| 733 |
+
self,
|
| 734 |
+
results: list[dict[str, Any]],
|
| 735 |
+
output_dir: Path,
|
| 736 |
+
) -> None:
|
| 737 |
+
"""حفظ مجموعة البيانات المُعزَّزة في مجلد.
|
| 738 |
+
|
| 739 |
+
يحفظ الصور كملفات PNG والتسميات كملف JSONL.
|
| 740 |
+
|
| 741 |
+
Args:
|
| 742 |
+
results: قائمة نتائج التعزيز.
|
| 743 |
+
output_dir: مجلد الحفظ.
|
| 744 |
+
"""
|
| 745 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 746 |
+
images_dir = output_dir / "images"
|
| 747 |
+
images_dir.mkdir(exist_ok=True)
|
| 748 |
+
|
| 749 |
+
label_data: list[dict[str, str]] = []
|
| 750 |
+
counter = 0
|
| 751 |
+
|
| 752 |
+
for result in results:
|
| 753 |
+
img = result.get("image")
|
| 754 |
+
label = result.get("label", "")
|
| 755 |
+
source = result.get("source_image", "unknown")
|
| 756 |
+
|
| 757 |
+
if img is None:
|
| 758 |
+
continue
|
| 759 |
+
|
| 760 |
+
try:
|
| 761 |
+
pil_img = self._to_pil(img) if self._has_numpy and not isinstance(img, Image.Image) else img
|
| 762 |
+
# تأكد من أنها PIL.Image
|
| 763 |
+
from PIL import Image
|
| 764 |
+
if not isinstance(pil_img, Image.Image):
|
| 765 |
+
pil_img = self._to_pil(img)
|
| 766 |
+
|
| 767 |
+
counter += 1
|
| 768 |
+
# اسم ملف: رقم + مصدر_أصلي
|
| 769 |
+
stem = Path(source).stem if source else f"img"
|
| 770 |
+
img_filename = f"{counter:06d}_{stem}.png"
|
| 771 |
+
img_path = images_dir / img_filename
|
| 772 |
+
|
| 773 |
+
pil_img.save(str(img_path), format="PNG")
|
| 774 |
+
|
| 775 |
+
label_data.append({
|
| 776 |
+
"image": f"images/{img_filename}",
|
| 777 |
+
"label": label,
|
| 778 |
+
"source": source,
|
| 779 |
+
})
|
| 780 |
+
|
| 781 |
+
except Exception as e:
|
| 782 |
+
logger.debug("فشل في حفظ صورة: %s", e)
|
| 783 |
+
|
| 784 |
+
# حفظ ملف التسميات
|
| 785 |
+
label_path = output_dir / "labels.jsonl"
|
| 786 |
+
with open(label_path, "w", encoding="utf-8") as f:
|
| 787 |
+
for entry in label_data:
|
| 788 |
+
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
| 789 |
+
|
| 790 |
+
logger.info(
|
| 791 |
+
"تم حفظ %d صورة في: %s و %d تسمية في: %s",
|
| 792 |
+
len(label_data), images_dir, len(label_data), label_path,
|
| 793 |
+
)
|
| 794 |
+
|
| 795 |
+
@staticmethod
|
| 796 |
+
def sanitize_arabic_label(label: str) -> str:
|
| 797 |
+
"""تنظيف التسمية العربية من التشكيل الزائد (اختياري).
|
| 798 |
+
|
| 799 |
+
مفيد لتوحيد التسميات عند المقارنة.
|
| 800 |
+
|
| 801 |
+
Args:
|
| 802 |
+
label: التسمية الأصلية.
|
| 803 |
+
|
| 804 |
+
Returns:
|
| 805 |
+
التسمية بدون تشكيل.
|
| 806 |
+
"""
|
| 807 |
+
diacritics = (
|
| 808 |
+
"\u0610\u0611\u0612\u0613\u0614\u0615\u0616"
|
| 809 |
+
"\u0617\u0618\u0619\u061A"
|
| 810 |
+
"\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652"
|
| 811 |
+
)
|
| 812 |
+
return "".join(c for c in label if c not in diacritics)
|
| 813 |
+
|
| 814 |
+
@staticmethod
|
| 815 |
+
def has_arabic_diacritics(text: str) -> bool:
|
| 816 |
+
"""فحص هل النص يحتوي تشكيلاً عربياً.
|
| 817 |
+
|
| 818 |
+
Args:
|
| 819 |
+
text: النص المراد فحصه.
|
| 820 |
+
|
| 821 |
+
Returns:
|
| 822 |
+
True إذا وُجد تشكيل.
|
| 823 |
+
"""
|
| 824 |
+
diacritics = (
|
| 825 |
+
"\u0610\u0611\u0612\u0613\u0614\u0615\u0616"
|
| 826 |
+
"\u0617\u0618\u0619\u061A"
|
| 827 |
+
"\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652"
|
| 828 |
+
)
|
| 829 |
+
return any(c in diacritics for c in text)
|
|
@@ -0,0 +1,597 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
مستخرج النصوص من الفيديو — Video OCR Module
|
| 3 |
+
================================================
|
| 4 |
+
استخراج النصوص من إطارات الفيديو مع طوابع زمنية باستخدام OpenCV + OCR.
|
| 5 |
+
|
| 6 |
+
القدرات:
|
| 7 |
+
- استخراج إطارات بمعدّل قابل للتهيئة (الافتراضي: كل 30 إطار ≈ 1 ثانية عند 30fps)
|
| 8 |
+
- معالجة ملفات الفيديو (mp4, avi, mov, mkv, webm)
|
| 9 |
+
- بث مباشر من الكاميرا
|
| 10 |
+
- نتائج مؤقتة (timestamped text) مع الثقة
|
| 11 |
+
- دعم رد الاتصال (callback) لتتبع التقدم
|
| 12 |
+
- تكامل كسول (lazy) مع محرك OCR الموجود
|
| 13 |
+
|
| 14 |
+
مثال الاستخدام:
|
| 15 |
+
>>> video_ocr = VideoOCR(frame_interval=30)
|
| 16 |
+
>>> # معالجة ملف فيديو
|
| 17 |
+
>>> results = video_ocr.process_video("lecture.mp4")
|
| 18 |
+
>>> for r in results:
|
| 19 |
+
... print(f"[{r['timestamp']}] {r['text']} (conf: {r['confidence']:.0%})")
|
| 20 |
+
>>> # بث مباشر من الكاميرا
|
| 21 |
+
>>> video_ocr.process_camera(camera_id=0, duration=10)
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import logging
|
| 27 |
+
import time
|
| 28 |
+
from dataclasses import dataclass, field
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
from typing import Any, Callable, Optional, Union
|
| 31 |
+
|
| 32 |
+
logger = logging.getLogger(__name__)
|
| 33 |
+
|
| 34 |
+
# صيغ الفيديو المدعومة
|
| 35 |
+
SUPPORTED_VIDEO_EXTENSIONS: set[str] = {
|
| 36 |
+
".mp4", ".avi", ".mov", ".mkv", ".webm", ".flv", ".wmv", ".m4v", ".mpg", ".mpeg",
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# ======================================================================
|
| 41 |
+
# هياكل البيانات
|
| 42 |
+
# ======================================================================
|
| 43 |
+
|
| 44 |
+
@dataclass
|
| 45 |
+
class FrameResult:
|
| 46 |
+
"""نتيجة OCR لإطار فيديو واحد.
|
| 47 |
+
|
| 48 |
+
Attributes:
|
| 49 |
+
frame_number: رقم الإطار في الفيديو.
|
| 50 |
+
timestamp: الطابع الزمني بالثواني (من بداية الفيديو).
|
| 51 |
+
timestamp_formatted: الطابع الزمني بتنسيق MM:SS.
|
| 52 |
+
text: النص المستخرج.
|
| 53 |
+
confidence: مستوى الثقة (0.0 - 1.0).
|
| 54 |
+
source_engine: محرك OCR المستخدم.
|
| 55 |
+
fps: معدّل الإطارات في الثانية.
|
| 56 |
+
"""
|
| 57 |
+
frame_number: int
|
| 58 |
+
timestamp: float
|
| 59 |
+
timestamp_formatted: str
|
| 60 |
+
text: str
|
| 61 |
+
confidence: float
|
| 62 |
+
source_engine: str = ""
|
| 63 |
+
fps: float = 0.0
|
| 64 |
+
metadata: dict[str, Any] = field(default_factory=dict)
|
| 65 |
+
|
| 66 |
+
def to_dict(self) -> dict[str, Any]:
|
| 67 |
+
"""تحويل إلى قاموس."""
|
| 68 |
+
return {
|
| 69 |
+
"frame_number": self.frame_number,
|
| 70 |
+
"timestamp": self.timestamp,
|
| 71 |
+
"timestamp_formatted": self.timestamp_formatted,
|
| 72 |
+
"text": self.text,
|
| 73 |
+
"confidence": self.confidence,
|
| 74 |
+
"source_engine": self.source_engine,
|
| 75 |
+
"fps": self.fps,
|
| 76 |
+
"metadata": self.metadata,
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@dataclass
|
| 81 |
+
class VideoTimeline:
|
| 82 |
+
"""الجدول الزمني للفيديو مع نتائج OCR.
|
| 83 |
+
|
| 84 |
+
يحتوي ملخصاً كاملاً لكل الإطارات المُعالجة.
|
| 85 |
+
"""
|
| 86 |
+
video_path: str = ""
|
| 87 |
+
total_frames: int = 0
|
| 88 |
+
processed_frames: int = 0
|
| 89 |
+
fps: float = 0.0
|
| 90 |
+
duration_seconds: float = 0.0
|
| 91 |
+
frame_interval: int = 30
|
| 92 |
+
results: list[FrameResult] = field(default_factory=list)
|
| 93 |
+
|
| 94 |
+
@property
|
| 95 |
+
def frames_with_text(self) -> int:
|
| 96 |
+
"""عدد الإطارات التي تحتوي نصاً."""
|
| 97 |
+
return sum(1 for r in self.results if r.text.strip())
|
| 98 |
+
|
| 99 |
+
@property
|
| 100 |
+
def avg_confidence(self) -> float:
|
| 101 |
+
"""متوسط الثقة للإطارات التي تحتوي نصاً."""
|
| 102 |
+
confidences = [r.confidence for r in self.results if r.text.strip() and r.confidence > 0]
|
| 103 |
+
return sum(confidences) / len(confidences) if confidences else 0.0
|
| 104 |
+
|
| 105 |
+
def to_dict(self) -> dict[str, Any]:
|
| 106 |
+
"""تحويل إلى قاموس."""
|
| 107 |
+
return {
|
| 108 |
+
"video_path": self.video_path,
|
| 109 |
+
"total_frames": self.total_frames,
|
| 110 |
+
"processed_frames": self.processed_frames,
|
| 111 |
+
"fps": self.fps,
|
| 112 |
+
"duration_seconds": round(self.duration_seconds, 2),
|
| 113 |
+
"frame_interval": self.frame_interval,
|
| 114 |
+
"frames_with_text": self.frames_with_text,
|
| 115 |
+
"avg_confidence": round(self.avg_confidence, 4),
|
| 116 |
+
"results": [r.to_dict() for r in self.results],
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
def get_text_only(self) -> str:
|
| 120 |
+
"""الحصول على النص الكامل فقط (بدون بيانات وصفية)."""
|
| 121 |
+
return "\n".join(
|
| 122 |
+
f"[{r.timestamp_formatted}] {r.text}"
|
| 123 |
+
for r in self.results
|
| 124 |
+
if r.text.strip()
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
# ======================================================================
|
| 129 |
+
# فئة مستخرج النصوص من الفيديو
|
| 130 |
+
# ======================================================================
|
| 131 |
+
|
| 132 |
+
class VideoOCR:
|
| 133 |
+
"""مستخرج النصوص من الفيديو باستخدام OpenCV و OCR.
|
| 134 |
+
|
| 135 |
+
يجمع بين استخراج الإطارات و التعرف على النصوص لتوفير
|
| 136 |
+
نتائج مؤقتة (timestamped) من محتوى الفيديو.
|
| 137 |
+
|
| 138 |
+
Attributes:
|
| 139 |
+
frame_interval: عدد الإطارات بين كل عملية استخراج.
|
| 140 |
+
ocr_engine: محرك OCR (يُحمَّل كسولاً عند أول استخدام).
|
| 141 |
+
"""
|
| 142 |
+
|
| 143 |
+
def __init__(
|
| 144 |
+
self,
|
| 145 |
+
frame_interval: int = 30,
|
| 146 |
+
ocr_engine: Optional[Any] = None,
|
| 147 |
+
confidence_threshold: float = 0.3,
|
| 148 |
+
min_text_length: int = 2,
|
| 149 |
+
max_results: Optional[int] = None,
|
| 150 |
+
) -> None:
|
| 151 |
+
"""تهيئة مستخرج النصوص من الفيديو.
|
| 152 |
+
|
| 153 |
+
Args:
|
| 154 |
+
frame_interval: عدد الإطارات بين كل عملية استخراج
|
| 155 |
+
(الافتراضي 30 ≈ 1 ثانية عند 30fps).
|
| 156 |
+
ocr_engine: مثيل OCREngine (اختياري، يُنشأ تلقائياً إذا لم يُحدد).
|
| 157 |
+
confidence_threshold: أقل حد للثقة لقبول النتيجة.
|
| 158 |
+
min_text_length: أقل طول نص مقبول (للتخلص من الضوضاء).
|
| 159 |
+
max_results: أقصى عدد نتائج (None = بلا حد).
|
| 160 |
+
"""
|
| 161 |
+
self.frame_interval = max(1, frame_interval)
|
| 162 |
+
self._ocr_engine = ocr_engine
|
| 163 |
+
self.confidence_threshold = confidence_threshold
|
| 164 |
+
self.min_text_length = min_text_length
|
| 165 |
+
self.max_results = max_results
|
| 166 |
+
|
| 167 |
+
# التحقق من توفر المكتبات
|
| 168 |
+
self._has_cv2 = self._check_library("cv2", "opencv-python")
|
| 169 |
+
self._has_pil = self._check_library("PIL", "Pillow")
|
| 170 |
+
|
| 171 |
+
if not self._has_cv2:
|
| 172 |
+
logger.warning(
|
| 173 |
+
"OpenCV غير مثبت. لن تعمل معالجة الفيديو. "
|
| 174 |
+
"pip install opencv-python-headless"
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
@staticmethod
|
| 178 |
+
def _check_library(import_name: str, package_name: str) -> bool:
|
| 179 |
+
"""التحقق من توفر مكتبة."""
|
| 180 |
+
try:
|
| 181 |
+
__import__(import_name)
|
| 182 |
+
return True
|
| 183 |
+
except ImportError:
|
| 184 |
+
return False
|
| 185 |
+
|
| 186 |
+
# ------------------------------------------------------------------
|
| 187 |
+
# تحميل محرك OCR (كسول)
|
| 188 |
+
# ------------------------------------------------------------------
|
| 189 |
+
|
| 190 |
+
def _get_ocr_engine(self) -> Any:
|
| 191 |
+
"""الحصول على محرك OCR (يُحمَّل عند أول استخدام).
|
| 192 |
+
|
| 193 |
+
Returns:
|
| 194 |
+
مثيل OCREngine أو None إذا لم يكن متاحاً.
|
| 195 |
+
"""
|
| 196 |
+
if self._ocr_engine is not None:
|
| 197 |
+
return self._ocr_engine
|
| 198 |
+
|
| 199 |
+
try:
|
| 200 |
+
from modules.vision.ocr_engine import OCREngine
|
| 201 |
+
self._ocr_engine = OCREngine(
|
| 202 |
+
enable_trocr=False,
|
| 203 |
+
enable_easyocr=True,
|
| 204 |
+
enable_tesseract=True,
|
| 205 |
+
enable_paddleocr=False,
|
| 206 |
+
enable_surya=False,
|
| 207 |
+
confidence_threshold=self.confidence_threshold,
|
| 208 |
+
preprocess=True,
|
| 209 |
+
)
|
| 210 |
+
logger.info("تم تحميل محرك OCR للفيديو بنجاح")
|
| 211 |
+
return self._ocr_engine
|
| 212 |
+
except Exception as e:
|
| 213 |
+
logger.error("فشل في تحميل محرك OCR: %s", e)
|
| 214 |
+
return None
|
| 215 |
+
|
| 216 |
+
# ------------------------------------------------------------------
|
| 217 |
+
# استخراج الإطارات
|
| 218 |
+
# ------------------------------------------------------------------
|
| 219 |
+
|
| 220 |
+
def extract_frames(
|
| 221 |
+
self,
|
| 222 |
+
video_path: str | Path,
|
| 223 |
+
output_dir: Optional[str | Path] = None,
|
| 224 |
+
progress_callback: Optional[Callable[[int, int], None]] = None,
|
| 225 |
+
) -> list[tuple[int, Any]]:
|
| 226 |
+
"""استخراج إطارات من ملف فيديو.
|
| 227 |
+
|
| 228 |
+
Args:
|
| 229 |
+
video_path: مسار ملف الفيديو.
|
| 230 |
+
output_dir: مجلد حفظ الإطارات (اختياري، لا يحفظ إذا None).
|
| 231 |
+
progress_callback: دالة (current_frame, total_frames) لتتبع التقدم.
|
| 232 |
+
|
| 233 |
+
Returns:
|
| 234 |
+
قائمة أزواج (frame_number, PIL.Image).
|
| 235 |
+
|
| 236 |
+
Raises:
|
| 237 |
+
FileNotFoundError: إذا لم يكن ملف الفيديو موجوداً.
|
| 238 |
+
RuntimeError: إذا لم يكن OpenCV متاحاً.
|
| 239 |
+
"""
|
| 240 |
+
video_path = Path(video_path)
|
| 241 |
+
if not video_path.exists():
|
| 242 |
+
raise FileNotFoundError(f"ملف الفيديو غير موجود: {video_path}")
|
| 243 |
+
|
| 244 |
+
if not self._has_cv2:
|
| 245 |
+
raise RuntimeError("OpenCV غير متاح — لا يمكن استخراج الإطارات")
|
| 246 |
+
|
| 247 |
+
import cv2
|
| 248 |
+
from PIL import Image
|
| 249 |
+
|
| 250 |
+
# التحقق من الصيغة
|
| 251 |
+
ext = video_path.suffix.lower()
|
| 252 |
+
if ext not in SUPPORTED_VIDEO_EXTENSIONS:
|
| 253 |
+
logger.warning("صيغة فيديو غير شائعة: %s — سيُحاول فتحها", ext)
|
| 254 |
+
|
| 255 |
+
# إنشاء مجلد الإطارات إذا طُلب
|
| 256 |
+
if output_dir:
|
| 257 |
+
output_dir = Path(output_dir)
|
| 258 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 259 |
+
|
| 260 |
+
cap = cv2.VideoCapture(str(video_path))
|
| 261 |
+
if not cap.isOpened():
|
| 262 |
+
raise RuntimeError(f"فشل في فتح ملف الفيديو: {video_path}")
|
| 263 |
+
|
| 264 |
+
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
| 265 |
+
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
| 266 |
+
|
| 267 |
+
frames: list[tuple[int, Image.Image]] = []
|
| 268 |
+
frame_number = 0
|
| 269 |
+
|
| 270 |
+
logger.info(
|
| 271 |
+
"جارٍ استخراج إطارات من: %s (إجمالي: %d, FPS: %.1f, فاصل: %d)",
|
| 272 |
+
video_path.name, total_frames, fps, self.frame_interval,
|
| 273 |
+
)
|
| 274 |
+
|
| 275 |
+
while True:
|
| 276 |
+
ret, frame = cap.read()
|
| 277 |
+
if not ret:
|
| 278 |
+
break
|
| 279 |
+
|
| 280 |
+
if frame_number % self.frame_interval == 0:
|
| 281 |
+
# تحويل BGR إلى RGB
|
| 282 |
+
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 283 |
+
pil_image = Image.fromarray(frame_rgb)
|
| 284 |
+
|
| 285 |
+
frames.append((frame_number, pil_image))
|
| 286 |
+
|
| 287 |
+
# حفظ الإطار إذا طُلب
|
| 288 |
+
if output_dir:
|
| 289 |
+
frame_path = output_dir / f"frame_{frame_number:06d}.png"
|
| 290 |
+
pil_image.save(frame_path)
|
| 291 |
+
|
| 292 |
+
# استدعاء رد الاتصال
|
| 293 |
+
if progress_callback:
|
| 294 |
+
try:
|
| 295 |
+
progress_callback(frame_number, total_frames)
|
| 296 |
+
except Exception as e:
|
| 297 |
+
logger.debug("خطأ في رد الاتصال: %s", e)
|
| 298 |
+
|
| 299 |
+
frame_number += 1
|
| 300 |
+
|
| 301 |
+
cap.release()
|
| 302 |
+
logger.info("تم استخراج %d إطار من %s", len(frames), video_path.name)
|
| 303 |
+
|
| 304 |
+
return frames
|
| 305 |
+
|
| 306 |
+
# ------------------------------------------------------------------
|
| 307 |
+
# معالجة الفيديو
|
| 308 |
+
# ------------------------------------------------------------------
|
| 309 |
+
|
| 310 |
+
def process_video(
|
| 311 |
+
self,
|
| 312 |
+
video_path: str | Path,
|
| 313 |
+
languages: Optional[list[str]] = None,
|
| 314 |
+
progress_callback: Optional[Callable[[int, int, float], None]] = None,
|
| 315 |
+
) -> VideoTimeline:
|
| 316 |
+
"""معالجة ملف فيديو واستخراج النصوص من إطاراته.
|
| 317 |
+
|
| 318 |
+
Args:
|
| 319 |
+
video_path: مسار ملف الفيديو.
|
| 320 |
+
languages: لغات OCR المطلوبة (مثل ["ar", "en"]).
|
| 321 |
+
progress_callback: دالة (frame_idx, total_frames, pct) لتتبع التقدم.
|
| 322 |
+
|
| 323 |
+
Returns:
|
| 324 |
+
كائن VideoTimeline يحتوي كل النتائج.
|
| 325 |
+
|
| 326 |
+
Raises:
|
| 327 |
+
FileNotFoundError: إذا لم يكن ملف الفيديو موجوداً.
|
| 328 |
+
"""
|
| 329 |
+
video_path = Path(video_path)
|
| 330 |
+
if not video_path.exists():
|
| 331 |
+
raise FileNotFoundError(f"ملف الفيديو غير موجود: {video_path}")
|
| 332 |
+
|
| 333 |
+
start_time = time.time()
|
| 334 |
+
|
| 335 |
+
# استخراج الإطارات
|
| 336 |
+
frames = self.extract_frames(video_path)
|
| 337 |
+
|
| 338 |
+
if not frames:
|
| 339 |
+
logger.warning("لم يُستخرج أي إطار من الفيديو")
|
| 340 |
+
return VideoTimeline(
|
| 341 |
+
video_path=str(video_path),
|
| 342 |
+
frame_interval=self.frame_interval,
|
| 343 |
+
)
|
| 344 |
+
|
| 345 |
+
# الحصول على محرك OCR
|
| 346 |
+
ocr_engine = self._get_ocr_engine()
|
| 347 |
+
if ocr_engine is None:
|
| 348 |
+
logger.warning("محرك OCR غير متاح — إرجاع إطارات بدون نص")
|
| 349 |
+
return VideoTimeline(
|
| 350 |
+
video_path=str(video_path),
|
| 351 |
+
total_frames=len(frames) * self.frame_interval,
|
| 352 |
+
processed_frames=len(frames),
|
| 353 |
+
frame_interval=self.frame_interval,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
# معالجة كل إطار
|
| 357 |
+
results: list[FrameResult] = []
|
| 358 |
+
total = len(frames)
|
| 359 |
+
|
| 360 |
+
# الحصول على FPS
|
| 361 |
+
fps = 30.0
|
| 362 |
+
if self._has_cv2:
|
| 363 |
+
import cv2
|
| 364 |
+
cap = cv2.VideoCapture(str(video_path))
|
| 365 |
+
if cap.isOpened():
|
| 366 |
+
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
| 367 |
+
cap.release()
|
| 368 |
+
|
| 369 |
+
for idx, (frame_num, image) in enumerate(frames):
|
| 370 |
+
try:
|
| 371 |
+
ocr_result = ocr_engine.recognize(image, languages=languages)
|
| 372 |
+
|
| 373 |
+
text = ocr_result.get("text", "").strip()
|
| 374 |
+
confidence = float(ocr_result.get("confidence", 0.0))
|
| 375 |
+
source = ocr_result.get("source", "unknown")
|
| 376 |
+
|
| 377 |
+
# فلترة النتائج
|
| 378 |
+
if len(text) < self.min_text_length:
|
| 379 |
+
text = ""
|
| 380 |
+
if confidence < self.confidence_threshold:
|
| 381 |
+
text = ""
|
| 382 |
+
|
| 383 |
+
timestamp = frame_num / fps if fps > 0 else 0.0
|
| 384 |
+
minutes = int(timestamp) // 60
|
| 385 |
+
seconds = int(timestamp) % 60
|
| 386 |
+
|
| 387 |
+
result = FrameResult(
|
| 388 |
+
frame_number=frame_num,
|
| 389 |
+
timestamp=round(timestamp, 2),
|
| 390 |
+
timestamp_formatted=f"{minutes:02d}:{seconds:02d}",
|
| 391 |
+
text=text,
|
| 392 |
+
confidence=confidence,
|
| 393 |
+
source_engine=source,
|
| 394 |
+
fps=fps,
|
| 395 |
+
metadata={
|
| 396 |
+
"processing_time": ocr_result.get("processing_time", 0.0),
|
| 397 |
+
},
|
| 398 |
+
)
|
| 399 |
+
results.append(result)
|
| 400 |
+
|
| 401 |
+
except Exception as e:
|
| 402 |
+
logger.warning("فشل OCR للإطار %d: %s", frame_num, e)
|
| 403 |
+
results.append(FrameResult(
|
| 404 |
+
frame_number=frame_num,
|
| 405 |
+
timestamp=round(frame_num / fps, 2) if fps > 0 else 0.0,
|
| 406 |
+
timestamp_formatted="00:00",
|
| 407 |
+
text="",
|
| 408 |
+
confidence=0.0,
|
| 409 |
+
fps=fps,
|
| 410 |
+
metadata={"error": str(e)},
|
| 411 |
+
))
|
| 412 |
+
|
| 413 |
+
# استدعاء رد الاتصال
|
| 414 |
+
if progress_callback:
|
| 415 |
+
try:
|
| 416 |
+
pct = (idx + 1) / total * 100
|
| 417 |
+
progress_callback(idx + 1, total, pct)
|
| 418 |
+
except Exception as e:
|
| 419 |
+
logger.debug("خطأ في رد الاتصال: %s", e)
|
| 420 |
+
|
| 421 |
+
# حد أقصى للنتائج
|
| 422 |
+
if self.max_results and len(results) >= self.max_results:
|
| 423 |
+
break
|
| 424 |
+
|
| 425 |
+
elapsed = time.time() - start_time
|
| 426 |
+
frames_with_text = sum(1 for r in results if r.text.strip())
|
| 427 |
+
|
| 428 |
+
logger.info(
|
| 429 |
+
"تمت معالجة الفيديو '%s': %d إطار, %d بنص, %.1f ثانية",
|
| 430 |
+
video_path.name, len(results), frames_with_text, elapsed,
|
| 431 |
+
)
|
| 432 |
+
|
| 433 |
+
timeline = VideoTimeline(
|
| 434 |
+
video_path=str(video_path),
|
| 435 |
+
total_frames=total * self.frame_interval,
|
| 436 |
+
processed_frames=len(frames),
|
| 437 |
+
fps=fps,
|
| 438 |
+
duration_seconds=total * self.frame_interval / fps if fps > 0 else 0.0,
|
| 439 |
+
frame_interval=self.frame_interval,
|
| 440 |
+
results=results,
|
| 441 |
+
)
|
| 442 |
+
|
| 443 |
+
return timeline
|
| 444 |
+
|
| 445 |
+
# ------------------------------------------------------------------
|
| 446 |
+
# معالجة الكاميرا (بث مباشر)
|
| 447 |
+
# ------------------------------------------------------------------
|
| 448 |
+
|
| 449 |
+
def process_camera(
|
| 450 |
+
self,
|
| 451 |
+
camera_id: int = 0,
|
| 452 |
+
duration: float = 10.0,
|
| 453 |
+
languages: Optional[list[str]] = None,
|
| 454 |
+
progress_callback: Optional[Callable[[int, int, float], None]] = None,
|
| 455 |
+
) -> list[FrameResult]:
|
| 456 |
+
"""معالجة بث مباشر من الكاميرا واستخراج النصوص.
|
| 457 |
+
|
| 458 |
+
Args:
|
| 459 |
+
camera_id: معرّف الكاميرا (الافتراضي 0 = الكاميرا الأساسية).
|
| 460 |
+
duration: مدة الالتقاط بالثواني.
|
| 461 |
+
languages: لغات OCR.
|
| 462 |
+
progress_callback: دالة (frame_idx, total_expected, pct) لتتبع التقدم.
|
| 463 |
+
|
| 464 |
+
Returns:
|
| 465 |
+
قائمة نتائج الإطارات.
|
| 466 |
+
"""
|
| 467 |
+
if not self._has_cv2:
|
| 468 |
+
raise RuntimeError("OpenCV غير متاح — لا يمكن الوصول للكاميرا")
|
| 469 |
+
|
| 470 |
+
import cv2
|
| 471 |
+
from PIL import Image
|
| 472 |
+
|
| 473 |
+
ocr_engine = self._get_ocr_engine()
|
| 474 |
+
if ocr_engine is None:
|
| 475 |
+
raise RuntimeError("محرك OCR غير متاح")
|
| 476 |
+
|
| 477 |
+
cap = cv2.VideoCapture(camera_id)
|
| 478 |
+
if not cap.isOpened():
|
| 479 |
+
raise RuntimeError(f"فشل في فتح الكاميرا {camera_id}")
|
| 480 |
+
|
| 481 |
+
fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
| 482 |
+
max_frames = int(duration * fps)
|
| 483 |
+
|
| 484 |
+
results: list[FrameResult] = []
|
| 485 |
+
frame_number = 0
|
| 486 |
+
start_time = time.time()
|
| 487 |
+
|
| 488 |
+
logger.info(
|
| 489 |
+
"جارٍ الالتقاط من الكاميرا %d (مدة: %.1fs, FPS: %.1f)",
|
| 490 |
+
camera_id, duration, fps,
|
| 491 |
+
)
|
| 492 |
+
|
| 493 |
+
try:
|
| 494 |
+
while frame_number < max_frames:
|
| 495 |
+
ret, frame = cap.read()
|
| 496 |
+
if not ret:
|
| 497 |
+
break
|
| 498 |
+
|
| 499 |
+
if frame_number % self.frame_interval == 0:
|
| 500 |
+
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 501 |
+
pil_image = Image.fromarray(frame_rgb)
|
| 502 |
+
|
| 503 |
+
try:
|
| 504 |
+
ocr_result = ocr_engine.recognize(pil_image, languages=languages)
|
| 505 |
+
|
| 506 |
+
text = ocr_result.get("text", "").strip()
|
| 507 |
+
confidence = float(ocr_result.get("confidence", 0.0))
|
| 508 |
+
|
| 509 |
+
if len(text) < self.min_text_length:
|
| 510 |
+
text = ""
|
| 511 |
+
if confidence < self.confidence_threshold:
|
| 512 |
+
text = ""
|
| 513 |
+
|
| 514 |
+
elapsed = time.time() - start_time
|
| 515 |
+
minutes = int(elapsed) // 60
|
| 516 |
+
seconds = int(elapsed) % 60
|
| 517 |
+
|
| 518 |
+
results.append(FrameResult(
|
| 519 |
+
frame_number=frame_number,
|
| 520 |
+
timestamp=round(elapsed, 2),
|
| 521 |
+
timestamp_formatted=f"{minutes:02d}:{seconds:02d}",
|
| 522 |
+
text=text,
|
| 523 |
+
confidence=confidence,
|
| 524 |
+
source_engine=ocr_result.get("source", "unknown"),
|
| 525 |
+
fps=fps,
|
| 526 |
+
))
|
| 527 |
+
except Exception as e:
|
| 528 |
+
logger.warning("فشل OCR للإطار %d من الكاميرا: %s", frame_number, e)
|
| 529 |
+
|
| 530 |
+
# استدعاء رد الاتصال
|
| 531 |
+
if progress_callback:
|
| 532 |
+
try:
|
| 533 |
+
pct = frame_number / max_frames * 100
|
| 534 |
+
progress_callback(frame_number, max_frames, pct)
|
| 535 |
+
except Exception:
|
| 536 |
+
pass
|
| 537 |
+
|
| 538 |
+
frame_number += 1
|
| 539 |
+
finally:
|
| 540 |
+
cap.release()
|
| 541 |
+
|
| 542 |
+
logger.info(
|
| 543 |
+
"تم الالتقاط من الكاميرا: %d إطار في %.1fs",
|
| 544 |
+
len(results), time.time() - start_time,
|
| 545 |
+
)
|
| 546 |
+
|
| 547 |
+
return results
|
| 548 |
+
|
| 549 |
+
# ------------------------------------------------------------------
|
| 550 |
+
# الجدول الزمني
|
| 551 |
+
# ------------------------------------------------------------------
|
| 552 |
+
|
| 553 |
+
def get_timeline(
|
| 554 |
+
self,
|
| 555 |
+
video_path: str | Path,
|
| 556 |
+
languages: Optional[list[str]] = None,
|
| 557 |
+
progress_callback: Optional[Callable[[int, int, float], None]] = None,
|
| 558 |
+
) -> dict[str, Any]:
|
| 559 |
+
"""الحصول على الجدول الزمني للنصوص في الفيديو.
|
| 560 |
+
|
| 561 |
+
يُرجع ملخصاً منسقاً مع النصوص المؤقتة.
|
| 562 |
+
|
| 563 |
+
Args:
|
| 564 |
+
video_path: مسار ملف الفيديو.
|
| 565 |
+
languages: لغات OCR.
|
| 566 |
+
progress_callback: دالة لتتبع التقدم.
|
| 567 |
+
|
| 568 |
+
Returns:
|
| 569 |
+
قاموس يحتوي:
|
| 570 |
+
- summary: ملخص إحصائي
|
| 571 |
+
- timeline: قائمة الإطارات مع النصوص
|
| 572 |
+
- full_text: النص الكامل فقط
|
| 573 |
+
"""
|
| 574 |
+
timeline = self.process_video(video_path, languages, progress_callback)
|
| 575 |
+
|
| 576 |
+
return {
|
| 577 |
+
"summary": {
|
| 578 |
+
"video_path": str(video_path),
|
| 579 |
+
"duration_seconds": timeline.duration_seconds,
|
| 580 |
+
"total_frames_processed": timeline.processed_frames,
|
| 581 |
+
"frames_with_text": timeline.frames_with_text,
|
| 582 |
+
"avg_confidence": timeline.avg_confidence,
|
| 583 |
+
"frame_interval": timeline.frame_interval,
|
| 584 |
+
},
|
| 585 |
+
"timeline": [
|
| 586 |
+
{
|
| 587 |
+
"timestamp": r.timestamp_formatted,
|
| 588 |
+
"timestamp_seconds": r.timestamp,
|
| 589 |
+
"text": r.text,
|
| 590 |
+
"confidence": r.confidence,
|
| 591 |
+
"engine": r.source_engine,
|
| 592 |
+
}
|
| 593 |
+
for r in timeline.results
|
| 594 |
+
if r.text.strip()
|
| 595 |
+
],
|
| 596 |
+
"full_text": timeline.get_text_only(),
|
| 597 |
+
}
|