Create src/preprocess_caselaw.py

src/preprocess_caselaw.py

@@ -0,0 +1,21 @@
+from datasets import load_dataset
+import json, os
+os.makedirs("data", exist_ok=True)
+
+def chunk_text(text, size=1000):
+    return [text[i:i+size] for i in range(0, len(text), size)]
+
+ds = load_dataset("common-pile/caselaw_access_project", split="train[:0.1%]")
+with open("data/caselaw_chunks.jsonl", "w") as f:
+    for item in ds:
+        text = item.get("text", "")
+        if len(text) < 200:
+            continue
+        for chunk in chunk_text(text):
+            f.write(json.dumps({
+                "case_name": item.get("case_name", ""),
+                "court": item.get("court", ""),
+                "text": chunk
+            }) + "\n")
+
+print("✅ Preprocessed and saved data/caselaw_chunks.jsonl")