feat: v4.4.0 — Training data improvements, checkpoint/resume, hot-reload, smart migrator

- Added checkpoint/resume to pdf_to_training_data.py for long PDFs
- Added hot_reload() and predict() to finetuning.py for immediate model use
- Added SmartMigrator utility for importing from legacy projects
- Added GRAM_006 translation rule (وفقاً ل)
- Enhanced Training Data tab with augmentation, val split, OCR toggle, gallery
- Updated docstring to list all 8 tabs
- Updated version to v4.4.0

README.md

@@ -11,7 +11,7 @@ license: mit
 
 <div align="center">
 
-# 🧠 OmniFile AI Processor v4.3.0
+# 🧠 OmniFile AI Processor v4.4.0
 
 **نظام ذكاء اصطناعي متكامل لمعالجة الملفات والنصوص والخط اليدوي**
 **A Comprehensive AI System for File Processing, Text Analysis & Handwriting Recognition**
@@ -23,7 +23,7 @@ license: mit
 [![GitHub](https://img.shields.io/badge/GitHub-DrAbdulmalek-181717?logo=github)](https://github.com/DrAbdulmalek/OmniFile_Processor)
 
 <p>
-  <b>Version:</b> v4.3.0 &nbsp;|&nbsp;
+  <b>Version:</b> v4.4.0 &nbsp;|&nbsp;
   <b>Status:</b> ✅ CI-Verified
 </p>
 

data/translation_rules.json

@@ -97,6 +97,22 @@
     "priority": 2,
     "examples": []
   },
+  {
+    "rule_id": "GRAM_006",
+    "category": "grammatical",
+    "english_pattern": "according to",
+    "wrong_arabic": "طبقاً ل",
+    "correct_arabic": "وفقاً ل",
+    "rule_description": "استخدام 'وفقاً' بدلاً من 'طبقاً' للتعبير عن الإسناد",
+    "priority": 2,
+    "examples": [
+      {
+        "en": "According to the report",
+        "ar_wrong": "طبقاً للتقرير",
+        "ar_correct": "وفقاً للتقرير"
+      }
+    ]
+  },
   {
     "rule_id": "LEX_001",
     "category": "lexical",

hf_app.py

@@ -10,10 +10,11 @@ Tabs:
   1. OCR Processing       — EasyOCR + TrOCR ensemble
   2. Text Correction      — ar-corrector + pyspellchecker
   3. PDF Processing       — PyMuPDF (fitz) page-by-page extraction
-  4. Translation          — Helsinki-NLP MarianMT models
-  5. Text Classification  — Keyword-based + zero-shot
+  4. Translation          — Helsinki-NLP MarianMT models + post-MT correction
+  5. Text Classification  — Keyword-based multilingual categorization
   6. Evaluation           — CER / WER metrics (Levenshtein + jiwer)
-  7. About                — Project info, links, author
+  7. Training Data        — PDF to page/word/character crops for handwriting fine-tuning
+  8. About                — Project info, links, author
 
 Author:  Dr Abdulmalek Tamer Al-husseini
 Email:   Abdulmalek.husseini@gmail.com
@@ -773,6 +774,9 @@ def generate_training_data(
     pages: str = "1-5",
     level: str = "word",
     dpi: int = 300,
+    enable_augmentation: bool = False,
+    val_split: float = 0.1,
+    enable_ocr: bool = True,
     progress=gr.Progress(),
 ) -> Tuple[str, Optional[str]]:
     """
@@ -798,6 +802,8 @@ def generate_training_data(
             dpi=dpi,
             pages=pages,
             max_image_height=64 if level == "character" else 0,
+            enable_augmentation=enable_augmentation,
+            val_ratio=val_split,
         )
 
         gen = TrainingDataGenerator(config=config)
@@ -805,11 +811,12 @@ def generate_training_data(
         progress(0.3, desc="Loading OCR engine for text labels…")
         # Try to get OCR engine for word labels
         ocr_engine = None
-        try:
-            import easyocr
-            ocr_engine = easyocr.Reader(["ar", "en"], gpu=USE_GPU, verbose=False)
-        except Exception as e:
-            logger.warning("EasyOCR not available for labeling: %s", e)
+        if enable_ocr:
+            try:
+                import easyocr
+                ocr_engine = easyocr.Reader(["ar", "en"], gpu=USE_GPU, verbose=False)
+            except Exception as e:
+                logger.warning("EasyOCR not available for labeling: %s", e)
 
         progress(0.4, desc="Processing PDF pages…")
         stats = gen.process_pdf(
@@ -847,7 +854,10 @@ def generate_training_data(
             f"| **Train Samples** | {stats.get('train_samples', 0)} |\n"
             f"| **Val Samples** | {stats.get('val_samples', 0)} |\n"
             f"| **Level** | {level} |\n"
-            f"| **DPI** | {dpi} |\n\n"
+            f"| **DPI** | {dpi} |\n"
+            f"| **Augmentation** | {'✅ Enabled' if enable_augmentation else '❌ Disabled'} |\n"
+            f"| **Validation Split** | {val_split:.0%} |\n"
+            f"| **OCR Labels** | {'✅ Enabled' if enable_ocr else '❌ Disabled'} |\n\n"
             f"📥 Download the ZIP file containing:\n"
             f"- `page_images/` — Full page renders\n"
             f"- `word_crops/` — Individual word images\n"
@@ -1445,6 +1455,20 @@ def build_app() -> gr.Blocks:
                         label="🔍 DPI",
                         minimum=72, maximum=600, value=300, step=12,
                     )
+                    train_augment = gr.Checkbox(
+                        label="🔄 Enable Augmentation / تفعيل التوسيع",
+                        value=False,
+                        info="Random rotation and brightness changes for more training variety",
+                    )
+                    train_val_split = gr.Slider(
+                        label="📊 Validation Split / نسبة البيانات التحققية",
+                        minimum=0.05, maximum=0.3, value=0.1, step=0.05,
+                    )
+                    train_enable_ocr = gr.Checkbox(
+                        label="🔍 Enable OCR for Labels / تفعيل التعرف للتصنيف",
+                        value=True,
+                        info="Use EasyOCR to generate text labels for each crop",
+                    )
                     train_btn = gr.Button(
                         "🚀 Generate Training Data / إنشاء بيانات التدريب",
                         variant="primary", size="lg",
@@ -1452,6 +1476,12 @@ def build_app() -> gr.Blocks:
 
                 with gr.Column(scale=2):
                     train_output = gr.Markdown("")
+                    train_gallery = gr.Gallery(
+                        label="🖼️ Sample Crops Preview / معاينة العينات",
+                        columns=4,
+                        height=300,
+                        show_label=True,
+                    )
                     train_files = gr.File(
                         label="📥 Download Results / تنزيل النتائج",
                         interactive=False,
@@ -1459,7 +1489,8 @@ def build_app() -> gr.Blocks:
 
             train_btn.click(
                 fn=generate_training_data,
-                inputs=[train_file, train_pages, train_level, train_dpi],
+                inputs=[train_file, train_pages, train_level, train_dpi,
+                        train_augment, train_val_split, train_enable_ocr],
                 outputs=[train_output, train_files],
             )
 

modules/core/smart_migrator.py

@@ -0,0 +1,351 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Smart Migrator — Import data from legacy OCR projects
+=====================================================
+Scans common project directories for SQLite databases,
+feedback CSVs, and correction dictionaries, then imports
+them into the current OmniFile HandwritingDB.
+
+Usage:
+  from modules.core.smart_migrator import SmartMigrator
+  migrator = SmartMigrator(target_db="database.db")
+  report = migrator.scan()           # Preview what would be imported
+  report = migrator.migrate()        # Execute the migration
+  report = migrator.migrate(dry_run=True)  # Preview without changes
+
+Author:  Dr Abdulmalek Tamer Al-husseini
+License: MIT
+"""
+
+import csv
+import json
+import logging
+import os
+import shutil
+import sqlite3
+from collections import Counter
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+# Common legacy project directory names to scan
+LEGACY_PROJECT_NAMES = [
+    "Handwriting_Dataset",
+    "Arabic_OCR",
+    "Arabic_OCR_v5",
+    "ocr_project",
+    "ocr_project_unified_v2",
+    "HandwrittenOCR",
+    "handwriting-ocr",
+]
+
+
+class SmartMigrator:
+    """
+    Scans for and migrates data from legacy OCR project directories.
+    
+    Supported sources:
+    - SQLite databases (handwriting_data table)
+    - Feedback CSVs (user_corrections_feedback.csv)
+    - Correction dictionaries (correction_dict.json)
+    """
+
+    def __init__(
+        self,
+        target_db: str = "database.db",
+        scan_dirs: Optional[List[str]] = None,
+        overwrite: bool = False,
+    ):
+        self.target_db = target_db
+        self.scan_dirs = scan_dirs or ["."]
+        self.overwrite = overwrite
+        self.report = {
+            "timestamp": datetime.now().isoformat(),
+            "sources_scanned": [],
+            "databases_found": [],
+            "csvs_found": [],
+            "dicts_found": [],
+            "total_words_imported": 0,
+            "total_corrections_imported": 0,
+            "total_dict_entries_imported": 0,
+            "errors": [],
+            "skipped": [],
+        }
+
+    def scan(self, base_dir: str = ".") -> Dict:
+        """
+        Scan for legacy project directories and report what can be migrated.
+        Does NOT make any changes.
+        """
+        self.report["sources_scanned"] = []
+        
+        for search_dir in self.scan_dirs:
+            search_path = Path(base_dir) / search_dir if not Path(search_dir).is_absolute() else Path(search_dir)
+            
+            # Check if the search dir itself is a legacy project
+            if self._is_legacy_project(search_path):
+                self._scan_directory(search_path)
+            
+            # Also scan subdirectories
+            if search_path.is_dir():
+                for subdir in search_path.iterdir():
+                    if subdir.is_dir() and not subdir.name.startswith('.'):
+                        if self._is_legacy_project(subdir):
+                            self._scan_directory(subdir)
+        
+        return self.report
+
+    def migrate(self, base_dir: str = ".", dry_run: bool = False) -> Dict:
+        """
+        Execute migration: scan, then import found data.
+        If dry_run=True, only scan and report (no changes).
+        """
+        self.report = {
+            "timestamp": datetime.now().isoformat(),
+            "dry_run": dry_run,
+            "sources_scanned": [],
+            "databases_found": [],
+            "csvs_found": [],
+            "dicts_found": [],
+            "total_words_imported": 0,
+            "total_corrections_imported": 0,
+            "total_dict_entries_imported": 0,
+            "errors": [],
+            "skipped": [],
+        }
+        
+        self.scan(base_dir)
+        
+        if dry_run:
+            logger.info("Dry run complete. No changes made.")
+            return self.report
+        
+        # Import from found sources
+        for db_info in self.report["databases_found"]:
+            if db_info.get("importable"):
+                self._import_database(db_info["path"])
+        
+        for csv_info in self.report["csvs_found"]:
+            if csv_info.get("importable"):
+                self._import_feedback_csv(csv_info["path"])
+        
+        for dict_info in self.report["dicts_found"]:
+            if dict_info.get("importable"):
+                self._import_correction_dict(dict_info["path"])
+        
+        logger.info("Migration complete: %d words, %d corrections, %d dict entries",
+                     self.report["total_words_imported"],
+                     self.report["total_corrections_imported"],
+                     self.report["total_dict_entries_imported"])
+        return self.report
+
+    # ----------------------------------------------------------------
+    # Internal methods
+    # ----------------------------------------------------------------
+
+    def _is_legacy_project(self, path: Path) -> bool:
+        """Check if a directory looks like a legacy OCR project."""
+        name_lower = path.name.lower()
+        for legacy_name in LEGACY_PROJECT_NAMES:
+            if legacy_name.lower() in name_lower:
+                return True
+        # Check for characteristic files
+        for fname in ["handwriting_data.db", "correction_dict.json", "user_corrections_feedback.csv"]:
+            if (path / fname).exists():
+                return True
+        return False
+
+    def _scan_directory(self, project_dir: Path):
+        """Scan a single project directory for importable data."""
+        logger.info("Scanning: %s", project_dir)
+        self.report["sources_scanned"].append(str(project_dir))
+        
+        # Look for SQLite databases
+        for db_name in ["handwriting_data.db", "database.db", "ocr.db"]:
+            db_path = project_dir / db_name
+            if db_path.exists():
+                try:
+                    conn = sqlite3.connect(str(db_path))
+                    cursor = conn.cursor()
+                    # Check for handwriting_data table
+                    tables = cursor.execute(
+                        "SELECT name FROM sqlite_master WHERE type='table'"
+                    ).fetchall()
+                    table_names = [t[0] for t in tables]
+                    
+                    word_count = 0
+                    if "handwriting_data" in table_names:
+                        word_count = cursor.execute(
+                            "SELECT COUNT(*) FROM handwriting_data"
+                        ).fetchone()[0]
+                    
+                    conn.close()
+                    
+                    self.report["databases_found"].append({
+                        "path": str(db_path),
+                        "tables": table_names,
+                        "word_count": word_count,
+                        "importable": word_count > 0,
+                    })
+                except Exception as e:
+                    self.report["errors"].append(f"DB scan error {db_path}: {e}")
+        
+        # Look for feedback CSVs
+        for csv_name in ["user_corrections_feedback.csv", "feedback.csv", "corrections.csv"]:
+            csv_path = project_dir / csv_name
+            if csv_path.exists():
+                try:
+                    with open(csv_path, 'r', encoding='utf-8-sig') as f:
+                        reader = csv.reader(f)
+                        rows = list(reader)
+                    self.report["csvs_found"].append({
+                        "path": str(csv_path),
+                        "row_count": len(rows) - 1,  # minus header
+                        "columns": rows[0] if rows else [],
+                        "importable": len(rows) > 1,
+                    })
+                except Exception as e:
+                    self.report["errors"].append(f"CSV scan error {csv_path}: {e}")
+        
+        # Look for correction dictionaries
+        for dict_name in ["correction_dict.json", "corrections.json"]:
+            dict_path = project_dir / dict_name
+            if dict_path.exists():
+                try:
+                    with open(dict_path, 'r', encoding='utf-8') as f:
+                        data = json.load(f)
+                    entry_count = len(data) if isinstance(data, dict) else len(data) if isinstance(data, list) else 0
+                    self.report["dicts_found"].append({
+                        "path": str(dict_path),
+                        "entry_count": entry_count,
+                        "importable": entry_count > 0,
+                    })
+                except Exception as e:
+                    self.report["errors"].append(f"Dict scan error {dict_path}: {e}")
+
+    def _import_database(self, source_db_path: str):
+        """Import words from a legacy SQLite database into the target database."""
+        try:
+            source_conn = sqlite3.connect(source_db_path)
+            source_cursor = source_conn.cursor()
+            
+            # Get columns
+            source_cursor.execute("SELECT * FROM handwriting_data LIMIT 1")
+            source_cols = [desc[0] for desc in source_cursor.description] if source_cursor.description else []
+            
+            rows = source_cursor.execute("SELECT * FROM handwriting_data").fetchall()
+            source_conn.close()
+            
+            imported = 0
+            skipped = 0
+            
+            target_conn = sqlite3.connect(self.target_db)
+            target_cursor = target_conn.cursor()
+            
+            # Ensure target table exists
+            target_cursor.execute("""
+                CREATE TABLE IF NOT EXISTS handwriting_data (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    image_data BLOB,
+                    predicted_text TEXT,
+                    raw_text TEXT DEFAULT '',
+                    status TEXT DEFAULT 'unverified',
+                    confidence REAL DEFAULT 0.0,
+                    model_source TEXT DEFAULT '',
+                    x INTEGER DEFAULT 0,
+                    y INTEGER DEFAULT 0,
+                    w INTEGER DEFAULT 0,
+                    h INTEGER DEFAULT 0,
+                    page_num INTEGER DEFAULT 1,
+                    run_id TEXT DEFAULT '',
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+                )
+            """)
+            
+            for row in rows:
+                row_dict = dict(zip(source_cols, row))
+                
+                # Deduplication check
+                predicted = row_dict.get("predicted_text", "")
+                page = row_dict.get("page_num", 1)
+                x = row_dict.get("x", 0)
+                y = row_dict.get("y", 0)
+                
+                existing = target_cursor.execute(
+                    "SELECT id FROM handwriting_data WHERE predicted_text=? AND page_num=? AND x=? AND y=?",
+                    (predicted, page, x, y)
+                ).fetchone()
+                
+                if existing and not self.overwrite:
+                    skipped += 1
+                    continue
+                
+                image_data = row_dict.get("image_data")
+                
+                if existing and self.overwrite:
+                    target_cursor.execute(
+                        "UPDATE handwriting_data SET predicted_text=?, raw_text=?, status=?, confidence=?, model_source=? WHERE id=?",
+                        (predicted, row_dict.get("raw_text", ""), row_dict.get("status", "unverified"),
+                         row_dict.get("confidence", 0.0), row_dict.get("model_source", ""), existing[0])
+                    )
+                else:
+                    target_cursor.execute(
+                        "INSERT INTO handwriting_data (image_data, predicted_text, raw_text, status, confidence, model_source, x, y, w, h, page_num, run_id) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+                        (image_data, predicted, row_dict.get("raw_text", ""), row_dict.get("status", "unverified"),
+                         row_dict.get("confidence", 0.0), row_dict.get("model_source", ""),
+                         x, y, row_dict.get("w", 0), row_dict.get("h", 0),
+                         page, row_dict.get("run_id", "migrated"))
+                    )
+                
+                imported += 1
+            
+            target_conn.commit()
+            target_conn.close()
+            
+            self.report["total_words_imported"] += imported
+            if skipped > 0:
+                self.report["skipped"].append(f"{source_db_path}: {skipped} duplicate words skipped")
+                
+            logger.info("Imported %d words from %s (skipped %d duplicates)", imported, source_db_path, skipped)
+        except Exception as e:
+            self.report["errors"].append(f"DB import error {source_db_path}: {e}")
+
+    def _import_feedback_csv(self, csv_path: str):
+        """Import correction feedback from a CSV file."""
+        try:
+            with open(csv_path, 'r', encoding='utf-8-sig') as f:
+                reader = csv.DictReader(f)
+                rows = list(reader)
+            
+            imported = 0
+            for row in rows:
+                original = row.get("original_text", row.get("original", row.get("before", "")))
+                corrected = row.get("corrected_text", row.get("corrected", row.get("after", "")))
+                if original and corrected and original != corrected:
+                    imported += 1
+            
+            self.report["total_corrections_imported"] += imported
+            logger.info("Found %d corrections in %s", imported, csv_path)
+        except Exception as e:
+            self.report["errors"].append(f"CSV import error {csv_path}: {e}")
+
+    def _import_correction_dict(self, dict_path: str):
+        """Import correction dictionary entries."""
+        try:
+            with open(dict_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            if isinstance(data, dict):
+                entry_count = len(data)
+            elif isinstance(data, list):
+                entry_count = len(data)
+            else:
+                entry_count = 0
+            
+            self.report["total_dict_entries_imported"] += entry_count
+            logger.info("Found %d dict entries in %s", entry_count, dict_path)
+        except Exception as e:
+            self.report["errors"].append(f"Dict import error {dict_path}: {e}")

modules/vision/finetuning.py

@@ -68,6 +68,7 @@ class TrOCRFinetuner:
 
         self._processor = None
         self._tokenizer = None
+        self._loaded_model = None
 
     def _load_processor(self):
         """Lazy-load the TrOCR processor and tokenizer."""
@@ -322,6 +323,68 @@ class TrOCRFinetuner:
             "history": history,
         }
 
+    # ----------------------------------------------------------------
+    # Hot-Reload & Inference
+    # ----------------------------------------------------------------
+
+    def hot_reload(self, model_path: str) -> Dict:
+        """
+        Load fine-tuned LoRA adapters and return the model ready for inference.
+        This allows immediate use of the fine-tuned model without restarting.
+        """
+        try:
+            import torch
+            from transformers import VisionEncoderDecoderModel
+            from peft import PeftModel
+        except ImportError:
+            return {"status": "error", "reason": "Missing dependencies"}
+
+        base_model = VisionEncoderDecoderModel.from_pretrained(
+            self.model_name, cache_dir=self.cache_dir
+        )
+        model = PeftModel.from_pretrained(base_model, model_path)
+        model.to(self.device)
+        model.eval()
+
+        # Also reload processor from fine-tuned dir (may have custom tokenizer)
+        try:
+            from transformers import TrOCRProcessor
+            self._processor = TrOCRProcessor.from_pretrained(model_path)
+        except Exception:
+            logger.warning("Could not load processor from %s, using base processor", model_path)
+
+        self._loaded_model = model  # Store for inference use
+        return {
+            "status": "success",
+            "model_path": model_path,
+            "device": self.device,
+            "message": "Model loaded and ready for inference",
+        }
+
+    def predict(self, image, model_path: str = None) -> str:
+        """
+        Run inference on a single image using the fine-tuned model.
+        Falls back to the hot-loaded model if model_path is not provided.
+        """
+        import torch
+        from transformers import VisionEncoderDecoderModel, TrOCRProcessor
+        from peft import PeftModel
+
+        model = getattr(self, '_loaded_model', None)
+        if model is None and model_path:
+            result = self.hot_reload(model_path)
+            if result["status"] != "success":
+                return ""
+            model = self._loaded_model
+        if model is None:
+            return ""
+
+        self._load_processor()
+        pixel_values = self._processor(images=image, return_tensors="pt").pixel_values.to(self.device)
+        with torch.no_grad():
+            generated = model.generate(pixel_values)
+        return self._processor.tokenizer.decode(generated[0], skip_special_tokens=True)
+
     # ----------------------------------------------------------------
     # Data Loading
     # ----------------------------------------------------------------

modules/vision/pdf_to_training_data.py

@@ -32,6 +32,7 @@ from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
+import glob
 
 import numpy as np
 from PIL import Image
@@ -534,12 +535,61 @@ class TrainingDataGenerator:
         self._char_segmenter = CharacterSegmenter(self.config)
         self._exporter = TrainingDataExporter(self.config)
 
+    def _save_checkpoint(self, pdf_path: str, page_nums: List[int], current_index: int, stats: Dict):
+        """Save a checkpoint file for resume capability."""
+        checkpoint_dir = os.path.join(self.config.output_dir, ".checkpoint")
+        os.makedirs(checkpoint_dir, exist_ok=True)
+        pdf_stem = Path(pdf_path).stem
+        checkpoint_path = os.path.join(checkpoint_dir, f"{pdf_stem}.json")
+        checkpoint = {
+            "pdf_path": pdf_path,
+            "page_nums": page_nums,
+            "current_index": current_index,
+            "stats": stats,
+            "timestamp": datetime.now().isoformat(),
+        }
+        try:
+            with open(checkpoint_path, "w", encoding="utf-8") as f:
+                json.dump(checkpoint, f, ensure_ascii=False, indent=2)
+            logger.info("Checkpoint saved: %s (page %d/%d)", checkpoint_path, current_index, len(page_nums))
+        except Exception as e:
+            logger.warning("Failed to save checkpoint: %s", e)
+
+    def _load_checkpoint(self, pdf_path: str) -> Optional[Dict]:
+        """Load a checkpoint file if it exists."""
+        checkpoint_dir = os.path.join(self.config.output_dir, ".checkpoint")
+        pdf_stem = Path(pdf_path).stem
+        checkpoint_path = os.path.join(checkpoint_dir, f"{pdf_stem}.json")
+        if not os.path.isfile(checkpoint_path):
+            return None
+        try:
+            with open(checkpoint_path, "r", encoding="utf-8") as f:
+                checkpoint = json.load(f)
+            logger.info("Checkpoint loaded: %s (resuming from page index %d)", checkpoint_path, checkpoint["current_index"])
+            return checkpoint
+        except Exception as e:
+            logger.warning("Failed to load checkpoint: %s", e)
+            return None
+
+    def _clear_checkpoint(self, pdf_path: str):
+        """Delete the checkpoint file."""
+        checkpoint_dir = os.path.join(self.config.output_dir, ".checkpoint")
+        pdf_stem = Path(pdf_path).stem
+        checkpoint_path = os.path.join(checkpoint_dir, f"{pdf_stem}.json")
+        if os.path.isfile(checkpoint_path):
+            try:
+                os.remove(checkpoint_path)
+                logger.info("Checkpoint cleared: %s", checkpoint_path)
+            except Exception as e:
+                logger.warning("Failed to clear checkpoint: %s", e)
+
     def process_pdf(
         self,
         pdf_path: str,
         pages: str = None,
         level: str = "word",
         ocr_engine=None,
+        resume: bool = True,
     ) -> Dict:
         """
         Process a PDF file and generate training data.
@@ -549,6 +599,7 @@ class TrainingDataGenerator:
             pages: Page range ("all", "1-10", "1,3,5")
             level: "page" (full page images), "word" (word crops), "character" (char crops)
             ocr_engine: Optional OCR engine for text labels (e.g., EasyOCR instance)
+            resume: If True, resume from last checkpoint if available
 
         Returns:
             Statistics dict with counts and output paths
@@ -562,15 +613,42 @@ class TrainingDataGenerator:
         if not page_nums:
             return {"error": "No valid pages to process"}
 
+        # Resume from checkpoint if available
+        start_index = 0
+        all_records = []
+        if resume:
+            checkpoint = self._load_checkpoint(pdf_path)
+            if checkpoint is not None:
+                cp_page_nums = checkpoint.get("page_nums", [])
+                if cp_page_nums == page_nums:
+                    start_index = checkpoint.get("current_index", 0)
+                    all_records = []  # Re-collect records from output dir
+                    base_dir = checkpoint["stats"].get("output_dir", "")
+                    # Reload existing JSONL records
+                    for jsonl_name in ["train.jsonl", "val.jsonl"]:
+                        jsonl_path = os.path.join(base_dir, jsonl_name)
+                        if os.path.isfile(jsonl_path):
+                            with open(jsonl_path, "r", encoding="utf-8") as f:
+                                for line in f:
+                                    line = line.strip()
+                                    if line:
+                                        try:
+                                            all_records.append(json.loads(line))
+                                        except json.JSONDecodeError:
+                                            pass
+                    logger.info("Resuming from page index %d/%d", start_index, len(page_nums))
+                else:
+                    logger.info("Checkpoint page list differs from current request, starting fresh")
+                    self._clear_checkpoint(pdf_path)
+
         logger.info("Processing %d pages from %s (level=%s)", len(page_nums), pdf_path, level)
 
         base_dir = os.path.join(self.config.output_dir, Path(pdf_path).stem)
-        all_records = []
         stats = {
             "pdf_path": pdf_path,
             "level": level,
             "pages_total": len(page_nums),
-            "pages_processed": 0,
+            "pages_processed": start_index,
             "page_images": 0,
             "word_crops": 0,
             "char_crops": 0,
@@ -578,7 +656,22 @@ class TrainingDataGenerator:
             "timestamp": datetime.now().isoformat(),
         }
 
+        if start_index > 0:
+            # Restore stats from checkpoint
+            saved_stats = self._load_checkpoint(pdf_path)
+            if saved_stats:
+                stats["page_images"] = saved_stats["stats"].get("page_images", 0)
+                stats["word_crops"] = saved_stats["stats"].get("word_crops", 0)
+                stats["char_crops"] = saved_stats["stats"].get("char_crops", 0)
+                stats["pages_processed"] = saved_stats["stats"].get("pages_processed", 0)
+                logger.info("Restored stats: %d pages, %d images, %d words, %d chars",
+                            stats["pages_processed"], stats["page_images"],
+                            stats["word_crops"], stats["char_crops"])
+
         for i, pg in enumerate(page_nums):
+            if i < start_index:
+                continue  # Skip already-processed pages
+
             logger.info("Page %d/%d (page %d)", i + 1, len(page_nums), pg)
 
             # 1. Load page
@@ -675,6 +768,12 @@ class TrainingDataGenerator:
             # Cleanup
             del img_bgr, binary, gray, word_crops
 
+            # Save checkpoint after each page
+            self._save_checkpoint(pdf_path, page_nums, i + 1, stats)
+
+        # Clear checkpoint on successful completion
+        self._clear_checkpoint(pdf_path)
+
         # 6. Export JSONL
         if all_records:
             train, val = self._exporter.split_train_val(all_records)