Upload datasets.ipynb

datasets.ipynb

@@ -9,18 +9,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from datasets import load_dataset\n",
     "import pandas as pd \n",
@@ -39,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -50,7 +41,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -60,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -71,7 +62,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -101,7 +92,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -151,7 +142,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -206,12 +197,96 @@
     "    return train_collection,test_collection\n",
     "\n",
     "# Train ve test datasetlerini MongoDB'ye yüklemek için fonksiyonu çağır\n",
-    "train_file_path = 'C:\\\\gitProjects\\\\bert\\\\datasets\\\\train_Egitim\\\\merged_train.parquet'\n",
-    "test_file_path = 'C:\\\\gitProjects\\\\bert\\\\datasets\\\\test_Egitim\\\\merged_test.parquet'\n",
+    "train_file_path = 'C:\\\\gitProjects\\\\yeni\\\\datasets\\\\train_Egitim\\\\merged_train.parquet'\n",
+    "test_file_path = 'C:\\\\gitProjects\\\\yeni\\\\datasets\\\\test_Egitim\\\\merged_test.parquet'\n",
     "\n",
     "train_collection, test_collection = dataset_read(train_file_path, test_file_path)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pymongo import MongoClient,errors\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sentence_transformers import SentenceTransformer\n",
+    "\n",
+    "# MongoDB bağlantı ve koleksiyon seçimi için fonksiyon\n",
+    "def get_mongodb(database_name='EgitimDatabase', train_collection_name='train', test_collection_name='test', host='localhost', port=27017):\n",
+    "    client = MongoClient(f'mongodb://{host}:{port}/')\n",
+    "    db = client[database_name]\n",
+    "    train_collection = db[train_collection_name]\n",
+    "    test_collection = db[test_collection_name]\n",
+    "    return train_collection, test_collection\n",
+    "\n",
+    "# Dataset'i MongoDB'ye yükleme fonksiyonu\n",
+    "def dataset_read(train_file_path, test_file_path):\n",
+    "    try:\n",
+    "        # MongoDB koleksiyonlarını al\n",
+    "        train_collection, test_collection = get_mongodb()\n",
+    "\n",
+    "        # Eğer koleksiyonlar zaten doluysa, veri yüklemesi yapma\n",
+    "        if train_collection.estimated_document_count() > 0 or test_collection.estimated_document_count() > 0:\n",
+    "            print(\"Veriler zaten yüklendi, işlem yapılmadı.\")\n",
+    "            return train_collection, test_collection\n",
+    "\n",
+    "        # Datasetleri oku\n",
+    "        data_train = pd.read_parquet(train_file_path, columns=['id', 'url', 'title', 'text'])\n",
+    "        data_test = pd.read_parquet(test_file_path, columns=['id', 'url', 'title', 'text'])\n",
+    "\n",
+    "        # Verileri MongoDB'ye yükle\n",
+    "        train_collection.insert_many(data_train.to_dict(\"records\"))\n",
+    "        test_collection.insert_many(data_test.to_dict(\"records\"))\n",
+    "\n",
+    "        print(f\"Veriler başarıyla {train_collection.name} koleksiyonuna yüklendi.\")\n",
+    "        print(f\"Veriler başarıyla {test_collection.name} koleksiyonuna yüklendi.\")\n",
+    "    \n",
+    "    except errors.PyMongoError as e:\n",
+    "        print(f\"Veri yükleme sırasında hata oluştu: {e}\")\n",
+    "\n",
+    "    return train_collection, test_collection\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Database sınıfı: Veritabanı bağlantıları ve verileri çekme işlevleri\n",
+    "class Database:\n",
+    "    @staticmethod\n",
+    "    def get_mongodb():\n",
+    "        return get_mongodb()\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def get_titles_and_texts():\n",
+    "        # MongoDB bağlantısı ve koleksiyonları al\n",
+    "        train_collection, _ = Database.get_mongodb()\n",
+    "\n",
+    "        # Sorgu: Hem \"title\" hem de \"text\" alanı mevcut olan belgeler\n",
+    "        query = {\"title\": {\"$exists\": True}, \"text\": {\"$exists\": True}}\n",
+    "\n",
+    "        # Belirtilen alanları seçiyoruz: \"title\", \"text\"\n",
+    "        cursor = train_collection.find(query, {\"title\": 1, \"text\": 1, \"_id\": 0})\n",
+    "\n",
+    "        # Başlık ve metinleri doğru bir şekilde birleştiriyoruz\n",
+    "        documents = [{\"title\": doc['title'], \"text\": doc['text']} for doc in cursor]\n",
+    "        document_count = len(documents)\n",
+    "        return documents, document_count\n",
+    "\n",
+    "# Train ve test datasetlerini MongoDB'ye yüklemek için fonksiyonu çağır\n",
+    "train_file_path = 'C:\\\\gitProjects\\\\yeni\\\\datasets\\\\train_Egitim\\\\merged_train.parquet'\n",
+    "test_file_path = 'C:\\\\gitProjects\\\\yeni\\\\datasets\\\\test_Egitim\\\\merged_test.parquet'\n",
+    "\n",
+    "train_collection, test_collection = dataset_read(train_file_path, test_file_path)\n",
+    "\n",
+    "# Veritabanından başlıklar ve metinler alınır\n",
+    "documents, document_count = Database.get_titles_and_texts()\n",
+    "\n",
+    "# Sonuçların belirlenmesi\n",
+    "print(f\"Başlık ve metin çiftleri: {documents}\")\n",
+    "print(f\"Toplam çift sayısı: {document_count}\")\n"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -225,80 +300,466 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "\"\"\"@staticmethod\n",
+    "    def get_input_titles():\n",
+    "        collection = Database.get_mongodb(collection_name='train')\n",
+    "        query = {\"title\": {\"$exists\": True}}\n",
+    "        cursor = collection.find(query, {\"title\": 1, \"_id\": 0})\n",
+    "        title_from_db = [doc['title'] for doc in cursor]\n",
+    "\n",
+    "        return title_from_db\"\"\"\n",
+    "\n",
+    "\"\"\"@staticmethod\n",
+    "    def get_input_texts():\n",
+    "        collection = Database.get_mongodb(collection_name='train')\n",
+    "        query = {\"texts\": {\"$exists\": True}}\n",
+    "        cursor = collection.find(query, {\"texts\": 1, \"_id\": 0})\n",
+    "        texts_from_db = [doc['texts'] for doc in cursor]\n",
+    "        return texts_from_db\"\"\"\n",
+    "    \n",
+    "    #bin tane veri çekerek csv dosyası olarak kaydetme \n",
+    "   \n",
+    "    \n",
+    "\"\"\"@staticmethod\n",
+    "    def get_titles_and_texts(batch_size=1000):\n",
+    "\n",
+    "        \n",
+    "        titles = Database.get_input_titles(batch_size=batch_size)\n",
+    "        texts = Database.get_input_texts(batch_size=batch_size )\n",
+    "        \n",
+    "\n",
+    "\n",
+    "    def test_queries():\n",
+    "\n",
+    "        collection = Database.get_mongodb(collection_name='train')\n",
+    "        # Başlık sorgusu\n",
+    "        titles_cursor = collection.find({\"title\": {\"$exists\": True}}, {\"title\": 1, \"_id\": 0})\n",
+    "        titles = [doc['title'] for doc in titles_cursor]\n",
+    "        \n",
+    "\n",
+    "        # Metin sorgusu\n",
+    "        texts_cursor = collection.find({\"text\": {\"$exists\": True}}, {\"text\": 1, \"_id\": 0})\n",
+    "        texts = [doc['text'] for doc in texts_cursor]\n",
+    "        \n",
+    "        # Başlık ve metinlerin eşleşmesini sağlamak için zip kullanarak birleştiriyoruz\n",
+    "        documents = [{\"title\": title, \"text\": text} for title, text in zip(titles, texts)]\n",
+    "        document_count = len(documents)\n",
+    "        return documents, document_count\n",
+    "\n",
+    "Database.test_queries()\n",
+    "\n",
+    "# Veritabanından başlıklar ve metinler alınır\n",
+    "documents, document_count = Database.get_titles_and_texts(batch_size=1000)\n",
+    "\n",
+    "# Sonuçların belirlenmesi\n",
+    "print(f\"Başlık ve metin çiftleri: {documents}\")\n",
+    "print(f\"Toplam çift sayısı: {document_count}\")\"\"\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Output'u vereceğimiz title ve textin kodu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0    **Pşıqo Ahecaqo** Pşıqo Ahecaqo (), Çerkes siy...\n",
+      "1    **Craterolophinae** Craterolophinae, Depastrid...\n",
+      "2    **Notocrabro** Notocrabro Crabronina oymağına ...\n",
+      "3    **Ibrahim Sissoko** İbrahim Sissoko (d. 30 Kas...\n",
+      "4    **Salah Cedid** Salah Cedid (1926-1993) (Arapç...\n",
+      "Name: combined, dtype: object\n",
+      "Veriler combined_output.csv dosyasına başarıyla kaydedildi.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pymongo import MongoClient\n",
+    "import pandas as pd\n",
+    "from tqdm.auto import tqdm, trange\n",
+    "\n",
+    "# Database bağlantıları ve verileri çekme işlevleri\n",
+    "class Database:\n",
+    "    @staticmethod\n",
+    "    def get_mongodb(database_name='EgitimDatabase', train_collection_name='train', test_collection_name='test', host='localhost', port=27017):\n",
+    "        client = MongoClient(f'mongodb://{host}:{port}/')\n",
+    "        db = client[database_name]\n",
+    "        train_collection = db[train_collection_name]\n",
+    "        test_collection = db[test_collection_name]\n",
+    "        return train_collection, test_collection\n",
+    "\n",
+    "    def export_to_csv(batch_size=1000, output_file='combined_output.csv'):\n",
+    "        train_collection, _ = Database.get_mongodb()\n",
+    "        cursor = train_collection.find({}, {\"title\": 1, \"text\": 1, \"_id\": 0})\n",
+    "        cursor = cursor.batch_size(batch_size)  # Fix: Call batch_size on the cursor object\n",
+    "\n",
+    "        # Verileri DataFrame'e dönüştürme\n",
+    "        df= pd.DataFrame(list(cursor))\n",
+    "        \n",
+    "        # title ve text sütunlarını birleştirme\n",
+    "        df['combined'] = df.apply(lambda row: f'**{row[\"title\"]}** {row[\"text\"]}', axis=1)\n",
+    "        \n",
+    "        #title,text and combined sütunlarını ayrı ayrı tutma\n",
+    "        #df2['title_only'] = df2['title']\n",
+    "        #df2['text_only'] = df2['text']\n",
+    "        #df['combined']= output_file\n",
+    "\n",
+    "        # Sonuçları kontrol etme\n",
+    "        combined_text= df['combined'] \n",
+    "          # Print the combined column directly\n",
+    "        \n",
+    "        print(combined_text.head())\n",
+    "\n",
+    "        # Birleşmiş verileri CSV'ye kaydetme\n",
+    "        \n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        \n",
+    "        print(f\"Veriler combined_output.csv dosyasına başarıyla kaydedildi.\")\n",
+    "        \n",
+    "\n",
+    "# CSV dosyasını okuma ve birleştirme işlemi\n",
+    "Database.export_to_csv()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "TF-IDF HESAPLAMA"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[nltk_data] Downloading package wordnet to\n",
+      "[nltk_data]     C:\\Users\\info\\AppData\\Roaming\\nltk_data...\n",
+      "[nltk_data]   Package wordnet is already up-to-date!\n",
+      "[nltk_data] Downloading package omw-1.4 to\n",
+      "[nltk_data]     C:\\Users\\info\\AppData\\Roaming\\nltk_data...\n",
+      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
+      "[nltk_data] Downloading package stopwords to\n",
+      "[nltk_data]     C:\\Users\\info\\AppData\\Roaming\\nltk_data...\n",
+      "[nltk_data]   Package stopwords is already up-to-date!\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "empty vocabulary; perhaps the documents only contain stop words",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[20], line 100\u001b[0m\n\u001b[0;32m     97\u001b[0m documents, document_count \u001b[38;5;241m=\u001b[39m Database\u001b[38;5;241m.\u001b[39mget_input_documents()\n\u001b[0;32m     99\u001b[0m \u001b[38;5;66;03m# Calculate TF-IDF and get feature names\u001b[39;00m\n\u001b[1;32m--> 100\u001b[0m tfidf_matrix, feature_names \u001b[38;5;241m=\u001b[39m \u001b[43mDatabase\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcalculate_tfidf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mturkish_stop_words\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    102\u001b[0m \u001b[38;5;66;03m# Extract keywords\u001b[39;00m\n\u001b[0;32m    103\u001b[0m keywords \u001b[38;5;241m=\u001b[39m Database\u001b[38;5;241m.\u001b[39mextract_keywords(tfidf_matrix, feature_names, stop_words\u001b[38;5;241m=\u001b[39mturkish_stop_words)\n",
+      "Cell \u001b[1;32mIn[20], line 43\u001b[0m, in \u001b[0;36mDatabase.calculate_tfidf\u001b[1;34m(documents, stop_words)\u001b[0m\n\u001b[0;32m     40\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[0;32m     41\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcalculate_tfidf\u001b[39m(documents, stop_words):\n\u001b[0;32m     42\u001b[0m     vectorizer \u001b[38;5;241m=\u001b[39m TfidfVectorizer(stop_words\u001b[38;5;241m=\u001b[39mstop_words, max_features\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10000\u001b[39m,min_df\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n\u001b[1;32m---> 43\u001b[0m     tfidf_matrix \u001b[38;5;241m=\u001b[39m \u001b[43mvectorizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     44\u001b[0m     feature_names \u001b[38;5;241m=\u001b[39m vectorizer\u001b[38;5;241m.\u001b[39mget_feature_names_out()\n\u001b[0;32m     45\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m tfidf_matrix, feature_names\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:2091\u001b[0m, in \u001b[0;36mTfidfVectorizer.fit_transform\u001b[1;34m(self, raw_documents, y)\u001b[0m\n\u001b[0;32m   2084\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_params()\n\u001b[0;32m   2085\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tfidf \u001b[38;5;241m=\u001b[39m TfidfTransformer(\n\u001b[0;32m   2086\u001b[0m     norm\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnorm,\n\u001b[0;32m   2087\u001b[0m     use_idf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_idf,\n\u001b[0;32m   2088\u001b[0m     smooth_idf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msmooth_idf,\n\u001b[0;32m   2089\u001b[0m     sublinear_tf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msublinear_tf,\n\u001b[0;32m   2090\u001b[0m )\n\u001b[1;32m-> 2091\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_documents\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   2092\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tfidf\u001b[38;5;241m.\u001b[39mfit(X)\n\u001b[0;32m   2093\u001b[0m \u001b[38;5;66;03m# X is already a transformed view of raw_documents so\u001b[39;00m\n\u001b[0;32m   2094\u001b[0m \u001b[38;5;66;03m# we set copy to False\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\base.py:1473\u001b[0m, in \u001b[0;36m_fit_context.<locals>.decorator.<locals>.wrapper\u001b[1;34m(estimator, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1466\u001b[0m     estimator\u001b[38;5;241m.\u001b[39m_validate_params()\n\u001b[0;32m   1468\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[0;32m   1469\u001b[0m     skip_parameter_validation\u001b[38;5;241m=\u001b[39m(\n\u001b[0;32m   1470\u001b[0m         prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[0;32m   1471\u001b[0m     )\n\u001b[0;32m   1472\u001b[0m ):\n\u001b[1;32m-> 1473\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m fit_method(estimator, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:1372\u001b[0m, in \u001b[0;36mCountVectorizer.fit_transform\u001b[1;34m(self, raw_documents, y)\u001b[0m\n\u001b[0;32m   1364\u001b[0m             warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[0;32m   1365\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUpper case characters found in\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1366\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m vocabulary while \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlowercase\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1367\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m is True. These entries will not\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1368\u001b[0m                 \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m be matched with any documents\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1369\u001b[0m             )\n\u001b[0;32m   1370\u001b[0m             \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m-> 1372\u001b[0m vocabulary, X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_count_vocab\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_documents\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfixed_vocabulary_\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1374\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbinary:\n\u001b[0;32m   1375\u001b[0m     X\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mfill(\u001b[38;5;241m1\u001b[39m)\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:1278\u001b[0m, in \u001b[0;36mCountVectorizer._count_vocab\u001b[1;34m(self, raw_documents, fixed_vocab)\u001b[0m\n\u001b[0;32m   1276\u001b[0m     vocabulary \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(vocabulary)\n\u001b[0;32m   1277\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m vocabulary:\n\u001b[1;32m-> 1278\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m   1279\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mempty vocabulary; perhaps the documents only contain stop words\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   1280\u001b[0m         )\n\u001b[0;32m   1282\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m indptr[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m>\u001b[39m np\u001b[38;5;241m.\u001b[39miinfo(np\u001b[38;5;241m.\u001b[39mint32)\u001b[38;5;241m.\u001b[39mmax:  \u001b[38;5;66;03m# = 2**31 - 1\u001b[39;00m\n\u001b[0;32m   1283\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m _IS_32BIT:\n",
+      "\u001b[1;31mValueError\u001b[0m: empty vocabulary; perhaps the documents only contain stop words"
+     ]
+    }
+   ],
+   "source": [
+    "#---------------------------güncel en yeni \n",
+    "from pymongo import MongoClient\n",
     "from sklearn.feature_extraction.text import TfidfVectorizer\n",
-    "from sentence_transformers import SentenceTransformer\n",
+    "from textblob import TextBlob as tb\n",
+    "import numpy as np\n",
+    "import math\n",
+    "from tqdm.auto import tqdm, trange\n",
+    "import tensorflow as tf\n",
+    "import nltk\n",
+    "from nltk.stem import WordNetLemmatizer\n",
+    "from nltk.corpus import stopwords\n",
     "\n",
-    "#bert base modeli \n",
-    "model = SentenceTransformer(\"emrecan/bert-base-turkish-cased-mean-nli-stsb-tr\")\n",
+    "turkish_stop_words = stopwords.words('turkish')\n",
+    "\n",
+    "nltk.download('wordnet')\n",
+    "nltk.download('omw-1.4')\n",
+    "nltk.download('stopwords')\n",
+    "\n",
+    "\n",
+    "import matplotlib.pyplot as plt \n",
     "\n",
-    "#text dosyasını koleksiyon üzerinden çekme \n",
-    "# Database sınıfı: Veritabanı bağlantıları ve verileri çekme işlevleri\n",
-    "# Database sınıfı: Veritabanı bağlantıları ve verileri çekme işlevleri\n",
     "class Database:\n",
     "    @staticmethod\n",
     "    def get_mongodb():\n",
-    "        # MongoDB bağlantı bilgilerini döndürecek şekilde tanımlanmıştır.\n",
-    "        return 'mongodb://localhost:27017/', 'EgitimDatabase', 'train'\n",
+    "        return 'mongodb://localhost:27017/', 'combined', 'combined_output'\n",
     "\n",
+    "    # Get input documents from MongoDB\n",
     "    @staticmethod\n",
-    "    def get_input_titles():\n",
+    "    def get_input_documents(limit=1000):\n",
     "        mongo_url, db_name, collection_name = Database.get_mongodb()\n",
     "        client = MongoClient(mongo_url)\n",
     "        db = client[db_name]\n",
     "        collection = db[collection_name]\n",
-    "        query = {\"title\": {\"$exists\": True}}\n",
-    "        cursor = collection.find(query, {\"title\": 1, \"_id\": 0})\n",
-    "        # Başlıkları listeye aldık\n",
-    "        title_from_db = [doc['title'] for doc in cursor]\n",
-    "        title_count = len(title_from_db)\n",
-    "        return title_from_db, title_count\n",
+    "        cursor = collection.find().limit(limit)\n",
+    "        combined_text = [doc['text'] for doc in cursor]\n",
+    "        document_count = len(combined_text)\n",
+    "        return combined_text, document_count\n",
     "    \n",
+    "    # Calculate TF-IDF and get feature names\n",
     "    @staticmethod\n",
-    "    def get_input_texts():\n",
-    "        mongo_url, db_name, collection_name = Database.get_mongodb()\n",
-    "        client = MongoClient(mongo_url)\n",
-    "        db = client[db_name]\n",
-    "        collection = db[collection_name]\n",
-    "        query = {\"text\": {\"$exists\": True}}\n",
-    "        cursor = collection.find(query, {\"text\": 1, \"_id\": 0})\n",
-    "        text_from_db = [doc['text'] for doc in cursor]\n",
-    "        text_count= len(text_from_db)\n",
-    "        return text_from_db,text_count\n",
+    "    def calculate_tfidf(documents, stop_words):\n",
+    "        vectorizer = TfidfVectorizer(stop_words=stop_words, max_features=10000,min_df=2)\n",
+    "        tfidf_matrix = vectorizer.fit_transform(documents)\n",
+    "        feature_names = vectorizer.get_feature_names_out()\n",
+    "        return tfidf_matrix, feature_names\n",
+    "\n",
+    "    # Extract keywords using TF-IDF\n",
+    "    def extract_keywords(tfidf_matrix, feature_names, top_n=10, stop_words=[]):\n",
+    "        keywords = {}\n",
+    "        for doc_idx, row in enumerate(tfidf_matrix):\n",
+    "            filtered_feature_names = [name for name in feature_names if name.lower() not in stop_words]\n",
+    "            scores = np.asarray(row.T.todense()).flatten()\n",
+    "            sorted_indices = np.argsort(scores)[::-1]\n",
+    "            top_features = sorted_indices[:top_n]\n",
+    "            doc_keywords = [(filtered_feature_names[idx], scores[idx]) for idx in top_features]\n",
+    "            keywords[f'document_{doc_idx+1}'] = doc_keywords\n",
+    "        return keywords\n",
+    "    \n",
+    "    #zip keywords and combined text \n",
+    "    \n",
+    "    # Identify low TF-IDF words\n",
+    "    @staticmethod\n",
+    "    def identify_low_tfidf_words(tfidf_matrix, feature_names, threshold=0.001):\n",
+    "        avg_scores = np.mean(tfidf_matrix, axis=0).A1\n",
+    "        low_tfidf_words = [feature_names[i] for i, score in enumerate(avg_scores) if score < threshold]\n",
+    "        return low_tfidf_words\n",
+    "    \n",
+    "    # Update stop words with low TF-IDF words\n",
+    "    @staticmethod\n",
+    "    def update_stop_words(existing_stop_words, low_tfidf_words):\n",
+    "        updated_stop_words = set(existing_stop_words) | set(low_tfidf_words)\n",
+    "        return list(updated_stop_words)\n",
     "\n",
     "\n",
-    "# Veritabanından başlıklar ve metinler alınır\n",
-    "titles, title_count = Database.get_input_titles()\n",
-    "texts = Database.get_input_texts()\n",
+    "#tf-ıdf ile döküman içerisinden kelime seçme \n",
+    "#Term Frequency (TF): Bir kelimenin belli bir dökümanda tekrar etme değeri\n",
+    "#Inverse Document Frequency (IDF):bir kelimenin tüm dökümanlar arasındaki yaygınlığı Nadir bulunan kelimeler, daha yüksek IDF değerine sahip olur.\n",
+    "#tf-ıdf skoru ise bu ikisinin çarpımıdır.\n",
     "\n",
-    "#sonuçların belirlenmesi\n",
-    "documents = titles + texts\n",
-    "print(f\"Başlıklar: {titles}\")\n",
-    "print(f\"Başlık sayısı: {title_count}\")\n",
-    "#print(f\"Metinler: {texts}\")\n",
-    "print(f\"Metin sayısı: {len(texts)}\")"
+    "    #buraya eşik değer belirlenmeli\n",
+    "\n",
+    "\n",
+    "turkish_stop_words = [\n",
+    "   'ah', 'ama', 'an', 'ancak', 'araba', 'aralar', 'aslında', \n",
+    "    'b', 'başlayan', 'bağlı', 'bazı', 'belirli', 'ben', 'bence', \n",
+    "    'birkaç', 'birlikte', 'bunu', 'burada', 'biten', 'biz', \n",
+    "    'bu', 'buna', 'çünkü', 'da', 'de', 'demek', 'den', 'derken', \n",
+    "    'değil', 'daha', 'dolayı', 'edilir', 'eğer', 'en', 'fakat', \n",
+    "    'genellikle', 'gibi', 'hem', 'her', 'herhangi', 'hiç', 'ise', \n",
+    "    'işte', 'itibaren', 'iyi', 'kadar', 'karşı', 'ki', 'kime', \n",
+    "    'kısaca', 'mu', 'mü', 'nasıl', 'ne', 'neden', 'niye', 'o', \n",
+    "    'olasılıkla', 'olabilir', 'oluşur', 'önce', 'şu', 'sadece', \n",
+    "    'se', 'şey', 'şimdi', 'tabi', 'tüm', 've', 'ya', 'ya da', \n",
+    "    'yanı', 'yani', 'yılında', 'yetenekli', 'yine'\n",
+    "]\n",
+    "# Get input documents\n",
+    "documents, document_count = Database.get_input_documents()\n",
+    "\n",
+    "# Calculate TF-IDF and get feature names\n",
+    "tfidf_matrix, feature_names = Database.calculate_tfidf(documents, turkish_stop_words)\n",
+    "\n",
+    "# Extract keywords\n",
+    "keywords = Database.extract_keywords(tfidf_matrix, feature_names, stop_words=turkish_stop_words)\n",
+    "print(keywords)\n",
+    "\n",
+    "# Identify low TF-IDF words\n",
+    "low_tfidf_words = Database.identify_low_tfidf_words(tfidf_matrix, feature_names)\n",
+    "print(low_tfidf_words)\n",
+    "\n",
+    "# Update stop words\n",
+    "updated_stop_words = Database.update_stop_words(turkish_stop_words, low_tfidf_words)\n",
+    "print(updated_stop_words) "
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": 15,
    "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "unhashable type: 'set'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[15], line 162\u001b[0m\n\u001b[0;32m    159\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m tfidf_matrix, feature_names,keywords\n\u001b[0;32m    161\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m\u001b[38;5;241m==\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 162\u001b[0m     tfidf_matrix, feature_names,keywords\u001b[38;5;241m=\u001b[39m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    164\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAnahtar Kelimler:\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    165\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m doc, words \u001b[38;5;129;01min\u001b[39;00m keywords\u001b[38;5;241m.\u001b[39mitems():\n",
+      "Cell \u001b[1;32mIn[15], line 148\u001b[0m, in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m    146\u001b[0m initial_stop_words \u001b[38;5;241m=\u001b[39m turkish_stop_words\n\u001b[0;32m    147\u001b[0m \u001b[38;5;66;03m# Stop-words listesini iteratif olarak güncelleyin\u001b[39;00m\n\u001b[1;32m--> 148\u001b[0m final_stop_words \u001b[38;5;241m=\u001b[39m \u001b[43miterative_update\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitial_stop_words\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    149\u001b[0m \u001b[38;5;66;03m#tf-ıdf hesaplama\u001b[39;00m\n\u001b[0;32m    150\u001b[0m tfidf_matrix, feature_names\u001b[38;5;241m=\u001b[39mcalculate_tfidf(documents_list,final_stop_words)\n",
+      "Cell \u001b[1;32mIn[15], line 127\u001b[0m, in \u001b[0;36miterative_update\u001b[1;34m(documents, initial_stop_words, iterations)\u001b[0m\n\u001b[0;32m    126\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21miterative_update\u001b[39m(documents, initial_stop_words, iterations\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m):\n\u001b[1;32m--> 127\u001b[0m     stop_words \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mset\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43minitial_stop_words\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    128\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(iterations):\n\u001b[0;32m    129\u001b[0m         tfidf_matrix, feature_names \u001b[38;5;241m=\u001b[39m calculate_tfidf(documents, stop_words)\n",
+      "\u001b[1;31mTypeError\u001b[0m: unhashable type: 'set'"
+     ]
+    }
+   ],
    "source": [
-    "TF-IDF HESAPLAMA"
+    "\n",
+    "\n",
+    "\"\"\"class Tf:\n",
+    "    @staticmethod\n",
+    "    def tf(word, blob):\n",
+    "        return blob.words.count(word) / len(blob.words)\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def n_containing(word, bloblist):\n",
+    "        return sum(1 for blob in bloblist if word in blob.words)\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def idf(word, bloblist):\n",
+    "        return math.log(len(bloblist) / (1 + Tf.n_containing(word, bloblist)))\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def tfidf(word, blob, bloblist):\n",
+    "        return Tf.tf(word, blob) * Tf.idf(word, bloblist)\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def get_input_documents(limit=1000):\n",
+    "        return Database.get_input_documents(limit)\"\"\"\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "    \"\"\"\n",
+    "    Her döküman için anahtar kelimeleri seç.\n",
+    "    :param tfidf_matrix: TF-IDF matris\n",
+    "    :param feature_names: TF-IDF özellik isimleri\n",
+    "    :param top_n: Her döküman için seçilecek anahtar kelime sayısı\n",
+    "    :return: Anahtar kelimeler ve skorlari\n",
+    "    \"\"\"\n",
+    "    \n",
+    "\n",
+    "#--------------------------------------------------------------- burada aldığımız  dökümanları listeliyoruz\n",
+    "# Dokümanları işleyerek TF-IDF hesaplama\n",
+    "#bloblist dökümanların bir listesi\n",
+    "\"\"\"bloblist = []\n",
+    "for i, blob in enumerate(bloblist):\n",
+    "    print(\"Top words in document {}\".format(i + 1))\n",
+    "    scores = {word: Tf.tfidf(word, blob, bloblist) for word in blob.words} #dökümanların içerisinde bulunan kelimeleri alır.\n",
+    "    sorted_words = sorted(scores.items(), key=lambda x: x[1], reverse=True)\n",
+    "    for word, score in sorted_words[:3]:\n",
+    "        print(\"\\tWord: {}, TF-IDF: {}\".format(word, round(score, 5)))\"\"\"\n",
+    "\n",
+    "\n",
+    "# Dökümanları isimlendir\n",
+    "#named_documents = {f'döküman {i+1}': doc for i, doc in enumerate(combined_text)}\n",
+    "\n",
+    "#features olarak top_keywordsleri belirleyerek metnin bu kelimelerin etrafında olması sağlanmalı  \n",
+    "def calculate_tfidf(documents, stop_words):\n",
+    "    vectorizer = TfidfVectorizer(stop_words=stop_words, max_features=10000)\n",
+    "    tfidf_matrix = vectorizer.fit_transform(documents)\n",
+    "    feature_names = vectorizer.get_feature_names_out()\n",
+    "    return tfidf_matrix, feature_names\n",
+    "\n",
+    "#---------------------------------------------------------------------------------\n",
+    "#kelimelerin ortalama skorlarını hesaplama \n",
+    "def identify_low_tfidf_words(tfidf_matrix, feature_names, threshold=0.001):\n",
+    "    # TF-IDF skorlarını toplayarak her kelimenin ortalama skorunu hesaplayın\n",
+    "    avg_scores = np.mean(tfidf_matrix, axis=0).A1\n",
+    "    low_tfidf_words = [feature_names[i] for i, score in enumerate(avg_scores) if score < threshold]\n",
+    "    return low_tfidf_words\n",
+    "\n",
+    "#kelimelerin yeni geliştirilen eşik değere göre güncellenmesi \n",
+    "def update_stop_words(existing_stop_words, low_tfidf_words):\n",
+    "    updated_stop_words = set(existing_stop_words) | set(low_tfidf_words)\n",
+    "    return list(updated_stop_words)\n",
+    "\n",
+    "\n",
+    "#bu kısım detaylandırılmalı \n",
+    "def iterative_update(documents, initial_stop_words, iterations=5):\n",
+    "    stop_words = set(initial_stop_words)\n",
+    "    for _ in range(iterations):\n",
+    "        tfidf_matrix, feature_names = calculate_tfidf(documents, stop_words)\n",
+    "        low_tfidf_words = identify_low_tfidf_words(tfidf_matrix, feature_names)\n",
+    "        stop_words = update_stop_words(stop_words, low_tfidf_words)\n",
+    "    return list(stop_words)\n",
+    "\n",
+    "\n",
+    "\n",
+    "def main ():\n",
+    "\n",
+    "    \n",
+    "#anlam ilişkisini de kontrol edecek bir yapı oluşpturulacak title ile benzerlik kontrol ederek yüksek benzerlik içeren kelimler sıralnacak .\n",
+    "\n",
+    "# Dökümanları liste olarak al\n",
+    "    named_documents, _ = Tf.get_input_documents(limit=1000)\n",
+    "    documents_list = [doc.get('text', '') if isinstance(doc, dict) else doc for doc in list(named_documents.values())]\n",
+    "\n",
+    "    #başlangıç stop değerleriyle yeni olanları arasında değişim yapma \n",
+    "    initial_stop_words = turkish_stop_words\n",
+    "    # Stop-words listesini iteratif olarak güncelleyin\n",
+    "    final_stop_words = iterative_update(documents_list, initial_stop_words)\n",
+    "    #tf-ıdf hesaplama\n",
+    "    tfidf_matrix, feature_names=calculate_tfidf(documents_list,final_stop_words)\n",
+    "    keywords=extract_keywords(tfidf_matrix,feature_names,top_n=10)\n",
+    "\n",
+    "    \n",
+    "\n",
+    "    print(\"Güncellenmiş Stop-Words Listesi:\", final_stop_words)\n",
+    "    print(\"TF-IDF Matrix Shape:\", tfidf_matrix.shape)\n",
+    "    print(\"Feature Names Sample:\", feature_names[:10])  # İlk 10 feature adını gösterir\n",
+    "\n",
+    "    return tfidf_matrix, feature_names,keywords\n",
+    "\n",
+    "if __name__==\"__main__\":\n",
+    "    tfidf_matrix, feature_names,keywords= main()\n",
+    "\n",
+    "    print(\"Anahtar Kelimler:\")\n",
+    "    for doc, words in keywords.items():\n",
+    "        print(f\"{doc}: {words}\")\n",
+    "    \n",
+    "\n",
+    "#---------------------------------------------------------\n",
+    "    \"\"\"blobs = [tb(doc) for doc in documents_list]  # veya 'title' kullanarak başlıkları işleyebilirsiniz\n",
+    "    all_words = set(word for blob in blobs for word in blob.words)\n",
+    "\n",
+    "    tfidf_scores = {}\n",
+    "    for word in all_words:\n",
+    "        tfidf_scores[word] = [Tf.tfidf(word, blob, blobs) for blob in blobs]\n",
+    "\n",
+    "    print(\"TF-IDF Skorları:\")\n",
+    "    for word, scores in tfidf_scores.items():\n",
+    "        print(f\"Kelime: {word}, Skorlar: {scores}\")\"\"\"\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "\"turkish_stop_words = set([\\n        'ad', 'adım', 'ah', 'ama', 'an', 'ancak', 'araba', 'aralar', 'aslında', \\n     'b', 'bazı', 'belirli', 'ben', 'bence', 'bunu', 'burada', 'biz', 'bu', 'buna', 'çünkü', \\n    'da', 'de', 'demek', 'den', 'derken', 'değil', 'daha', 'dolayı',  'edilir', 'eğer', 'en', 'fakat', \\n    'genellikle', 'gibi', 'hem', 'her', 'herhangi', 'hiç', 'ise', 'işte', 'itibaren', 'iyi', 'kadar', \\n    'karşı', 'ki', 'kime', 'kısaca', 'mu', 'mü', 'nasıl', 'ne', 'neden', 'niye', 'o', 'olabilir', 'oluşur', \\n    'önce', 'şu', 'sadece', 'se', 'şey', 'şimdi', 'tabi', 'tüm', 've', 'ya', 'ya da', 'yani', 'yine'\\n])\\ndef calculate_tfidf(documents):\\n    vectorizer = TfidfVectorizer(stop_words=turkish_stop_words, max_features=10000)  # max_features ile özellik sayısını sınırlıyoruz\\n    tfidf_matrix = vectorizer.fit_transform(documents)\\n    feature_names = vectorizer.get_feature_names_out()\\n    return tfidf_matrix, feature_names\\n\\n#feature_names lerin belirlenmesi grekir \\ntfidf_matrix, feature_names=calculate_tfidf(documents)\\n\\n\\n\\n# En yüksek TF-IDF skorlarına sahip anahtar kelimeleri çıkarın\\n#sıkışık format kullanmarak tf-ıdf matrisini işleme \\ndef get_top_n_keywords_sparse(n=10):\\n\\n    # TF-IDF hesaplayıcı oluşturun\\n    vectorizer = TfidfVectorizer()\\n\\n    # Başlıklar ve metinler ile TF-IDF matrisini oluşturun\\n    texts = Database.get_input_texts()\\n    titles = Database.get_input_titles()\\n    \\n\\n    #title ve text değerlerini alarak vektörleştirdik.\\n    tfidf_matrix = vectorizer.fit_transform(documents)\\n\\n    # Özellik adlarını (kelimeleri) alın\\n\\n    feature_names = vectorizer.get_feature_names_out()\\n\\n    # TF-IDF sonuçlarını DataFrame'e dönüştürün\\n    df = pd.DataFrame(tfidf_matrix.toarray(), columns=feature_names)\\n    print(df)\\n    keywords = {}\\n    for i in range(tfidf_matrix.shape[0]):\\n        row = tfidf_matrix[i].toarray().flatten() #list yapısından çıkarma \\n        sorted_indices = row.argsort()[::-1]  # Büyükten küçüğe sıralama\\n        top_indices = sorted_indices[:n]\\n        top_keywords = [feature_names[idx] for idx in top_indices]\\n        keywords[i] = top_keywords\\n    return keywords\""
-      ]
-     },
-     "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
+     "ename": "InvalidParameterError",
+     "evalue": "The 'stop_words' parameter of TfidfVectorizer must be a str among {'english'}, an instance of 'list' or None. Got {'o', 'den', 'an', 'şey', 'burada', 've', 'ah', 'ise', 'hiç', 'yine', 'biz', 'bu', 'da', 'genellikle', 'yılında', 'belirli', 'se', 'ne', 'kadar', 'neden', 'hem', 'aralar', 'yani', 'daha', 'araba', 'derken', 'dolayı', 'kısaca', 'karşı', 'niye', 'ki', 'bunu', 'buna', 'de', 'herhangi', 'önce', 'tabi', 'kime', 'biten', 'ben', 'ya', 'ya da', 'çünkü', 'mu', 'b', 'demek', 'fakat', 'şimdi', 'birlikte', 'her', 'bağlı', 'nasıl', 'şu', 'sadece', 'tüm', 'aslında', 'edilir', 'ama', 'bence', 'en', 'işte', 'gibi', 'ancak', 'birkaç', 'itibaren', 'mü', 'olabilir', 'bazı', 'oluşur', 'başlayan', 'yanı', 'olasılıkla', 'iyi', 'değil', 'eğer', 'yetenekli'} instead.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mInvalidParameterError\u001b[0m                     Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[2], line 155\u001b[0m\n\u001b[0;32m    152\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m tfidf_matrix, feature_names,documents_list \n\u001b[0;32m    154\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m\u001b[38;5;241m==\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 155\u001b[0m     tfidf_matrix, feature_names,documents_list\u001b[38;5;241m=\u001b[39m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    158\u001b[0m \u001b[38;5;66;03m# Sonuçları yazdır\u001b[39;00m\n\u001b[0;32m    159\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mİsimlendirilmiş Dökümanlar:\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "Cell \u001b[1;32mIn[2], line 142\u001b[0m, in \u001b[0;36mmain\u001b[1;34m()\u001b[0m\n\u001b[0;32m    140\u001b[0m initial_stop_words \u001b[38;5;241m=\u001b[39m turkish_stop_words\n\u001b[0;32m    141\u001b[0m \u001b[38;5;66;03m# Stop-words listesini iteratif olarak güncelleyin\u001b[39;00m\n\u001b[1;32m--> 142\u001b[0m final_stop_words \u001b[38;5;241m=\u001b[39m \u001b[43miterative_update\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments_list\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitial_stop_words\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    143\u001b[0m \u001b[38;5;66;03m#tf-ıdf hesaplama\u001b[39;00m\n\u001b[0;32m    144\u001b[0m tfidf_matrix, feature_names\u001b[38;5;241m=\u001b[39mcalculate_tfidf(documents_list,final_stop_words)\n",
+      "Cell \u001b[1;32mIn[2], line 124\u001b[0m, in \u001b[0;36miterative_update\u001b[1;34m(documents, initial_stop_words, iterations)\u001b[0m\n\u001b[0;32m    122\u001b[0m stop_words \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m(initial_stop_words)\n\u001b[0;32m    123\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(iterations):\n\u001b[1;32m--> 124\u001b[0m     tfidf_matrix, feature_names \u001b[38;5;241m=\u001b[39m \u001b[43mcalculate_tfidf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstop_words\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    125\u001b[0m     low_tfidf_words \u001b[38;5;241m=\u001b[39m identify_low_tfidf_words(tfidf_matrix, feature_names)\n\u001b[0;32m    126\u001b[0m     stop_words \u001b[38;5;241m=\u001b[39m update_stop_words(stop_words, low_tfidf_words)\n",
+      "Cell \u001b[1;32mIn[2], line 103\u001b[0m, in \u001b[0;36mcalculate_tfidf\u001b[1;34m(documents, stop_words)\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcalculate_tfidf\u001b[39m(documents, stop_words):\n\u001b[0;32m    102\u001b[0m     vectorizer \u001b[38;5;241m=\u001b[39m TfidfVectorizer(stop_words\u001b[38;5;241m=\u001b[39mstop_words, max_features\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10000\u001b[39m)\n\u001b[1;32m--> 103\u001b[0m     tfidf_matrix \u001b[38;5;241m=\u001b[39m \u001b[43mvectorizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdocuments\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    104\u001b[0m     feature_names \u001b[38;5;241m=\u001b[39m vectorizer\u001b[38;5;241m.\u001b[39mget_feature_names_out()\n\u001b[0;32m    105\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m tfidf_matrix, feature_names\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:2091\u001b[0m, in \u001b[0;36mTfidfVectorizer.fit_transform\u001b[1;34m(self, raw_documents, y)\u001b[0m\n\u001b[0;32m   2084\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_params()\n\u001b[0;32m   2085\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tfidf \u001b[38;5;241m=\u001b[39m TfidfTransformer(\n\u001b[0;32m   2086\u001b[0m     norm\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnorm,\n\u001b[0;32m   2087\u001b[0m     use_idf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_idf,\n\u001b[0;32m   2088\u001b[0m     smooth_idf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msmooth_idf,\n\u001b[0;32m   2089\u001b[0m     sublinear_tf\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msublinear_tf,\n\u001b[0;32m   2090\u001b[0m )\n\u001b[1;32m-> 2091\u001b[0m X \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit_transform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mraw_documents\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   2092\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tfidf\u001b[38;5;241m.\u001b[39mfit(X)\n\u001b[0;32m   2093\u001b[0m \u001b[38;5;66;03m# X is already a transformed view of raw_documents so\u001b[39;00m\n\u001b[0;32m   2094\u001b[0m \u001b[38;5;66;03m# we set copy to False\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\base.py:1466\u001b[0m, in \u001b[0;36m_fit_context.<locals>.decorator.<locals>.wrapper\u001b[1;34m(estimator, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1461\u001b[0m partial_fit_and_fitted \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m   1462\u001b[0m     fit_method\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpartial_fit\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m _is_fitted(estimator)\n\u001b[0;32m   1463\u001b[0m )\n\u001b[0;32m   1465\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m global_skip_validation \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m partial_fit_and_fitted:\n\u001b[1;32m-> 1466\u001b[0m     \u001b[43mestimator\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_params\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1468\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[0;32m   1469\u001b[0m     skip_parameter_validation\u001b[38;5;241m=\u001b[39m(\n\u001b[0;32m   1470\u001b[0m         prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[0;32m   1471\u001b[0m     )\n\u001b[0;32m   1472\u001b[0m ):\n\u001b[0;32m   1473\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m fit_method(estimator, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\base.py:666\u001b[0m, in \u001b[0;36mBaseEstimator._validate_params\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    658\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_validate_params\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    659\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Validate types and values of constructor parameters\u001b[39;00m\n\u001b[0;32m    660\u001b[0m \n\u001b[0;32m    661\u001b[0m \u001b[38;5;124;03m    The expected type and values must be defined in the `_parameter_constraints`\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    664\u001b[0m \u001b[38;5;124;03m    accepted constraints.\u001b[39;00m\n\u001b[0;32m    665\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 666\u001b[0m     \u001b[43mvalidate_parameter_constraints\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    667\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_parameter_constraints\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    668\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_params\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdeep\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    669\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcaller_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__class__\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__name__\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m    670\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sklearn\\utils\\_param_validation.py:95\u001b[0m, in \u001b[0;36mvalidate_parameter_constraints\u001b[1;34m(parameter_constraints, params, caller_name)\u001b[0m\n\u001b[0;32m     89\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m     90\u001b[0m     constraints_str \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m     91\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin([\u001b[38;5;28mstr\u001b[39m(c)\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mfor\u001b[39;00m\u001b[38;5;250m \u001b[39mc\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39mconstraints[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]])\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m or\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     92\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconstraints[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     93\u001b[0m     )\n\u001b[1;32m---> 95\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m InvalidParameterError(\n\u001b[0;32m     96\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mparam_name\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m parameter of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcaller_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m must be\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     97\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconstraints_str\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mparam_val\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m instead.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     98\u001b[0m )\n",
+      "\u001b[1;31mInvalidParameterError\u001b[0m: The 'stop_words' parameter of TfidfVectorizer must be a str among {'english'}, an instance of 'list' or None. Got {'o', 'den', 'an', 'şey', 'burada', 've', 'ah', 'ise', 'hiç', 'yine', 'biz', 'bu', 'da', 'genellikle', 'yılında', 'belirli', 'se', 'ne', 'kadar', 'neden', 'hem', 'aralar', 'yani', 'daha', 'araba', 'derken', 'dolayı', 'kısaca', 'karşı', 'niye', 'ki', 'bunu', 'buna', 'de', 'herhangi', 'önce', 'tabi', 'kime', 'biten', 'ben', 'ya', 'ya da', 'çünkü', 'mu', 'b', 'demek', 'fakat', 'şimdi', 'birlikte', 'her', 'bağlı', 'nasıl', 'şu', 'sadece', 'tüm', 'aslında', 'edilir', 'ama', 'bence', 'en', 'işte', 'gibi', 'ancak', 'birkaç', 'itibaren', 'mü', 'olabilir', 'bazı', 'oluşur', 'başlayan', 'yanı', 'olasılıkla', 'iyi', 'değil', 'eğer', 'yetenekli'} instead."
+     ]
     }
    ],
    "source": [
@@ -315,6 +776,8 @@
     "    def get_mongodb():\n",
     "        return 'mongodb://localhost:27017/', 'EgitimDatabase', 'train'\n",
     "\n",
+    "#--------------------------------------------------------------------------\n",
+    "#combined_text eklenmeli \n",
     "    @staticmethod\n",
     "    def get_input_documents(limit=3):\n",
     "        mongo_url, db_name, collection_name = Database.get_mongodb()\n",
@@ -322,11 +785,11 @@
     "        db = client[db_name]\n",
     "        collection = db[collection_name]\n",
     "        cursor = collection.find().limit(limit)\n",
-    "        documents = [doc for doc in cursor]\n",
-    "        document_count = len(documents)\n",
+    "        combined_text = [doc for doc in cursor]\n",
+    "        document_count = len(combined_text)\n",
     "        \n",
     "        # Dökümanları isimlendir\n",
-    "        named_documents = {f'döküman {i+1}': doc for i, doc in enumerate(documents)}\n",
+    "        named_documents = {f'döküman {i+1}': doc for i, doc in enumerate(combined_text)}\n",
     "        \n",
     "        return named_documents, document_count\n",
     "\n",
@@ -353,7 +816,7 @@
     "        return Database.get_input_documents(limit)\n",
     "\n",
     "# Kullanım örneği\n",
-    "named_documents, document_count = Tf.get_input_documents(limit=3)\n",
+    "named_documents, document_count = Tf.get_input_documents(limit=1000)\n",
     "\n",
     "#tf-ıdf ile döküman içerisinden kelime seçme \n",
     "\n",
@@ -387,23 +850,30 @@
     "    for word, score in sorted_words[:3]:\n",
     "        print(\"\\tWord: {}, TF-IDF: {}\".format(word, round(score, 5)))\n",
     "\n",
+    "\n",
+    "#buraya eşik değer belirlenmeli\n",
     "turkish_stop_words = [\n",
     "    'ah', 'ama', 'an', 'ancak', 'araba', 'aralar', 'aslında', \n",
-    "     'b','başlayan','bağlı', 'bazı', 'belirli', 'ben', 'bence','birkaç','birlikte', 'bunu', 'burada','biten','biten' ,'biz', 'bu', 'buna', 'çünkü', \n",
-    "    'da', 'de', 'demek', 'den', 'derken', 'değil', 'daha', 'dolayı',  'edilir', 'eğer', 'en', 'fakat', \n",
-    "    'genellikle', 'gibi', 'hem', 'her', 'herhangi', 'hiç', 'ise', 'işte', 'itibaren', 'iyi', 'kadar', \n",
-    "    'karşı', 'ki', 'kime', 'kısaca', 'mu', 'mü', 'nasıl', 'ne', 'neden', 'niye', 'o', 'olasılıkla','olabilir', 'oluşur', \n",
-    "    'önce', 'şu', 'sadece', 'se', 'şey', 'şimdi', 'tabi', 'tüm', 've', 'ya', 'ya da','yanı' ,'yanı','yani','yılında','yılında','yetenekli', 'yine'\n",
+    "    'b', 'başlayan', 'bağlı', 'bazı', 'belirli', 'ben', 'bence', \n",
+    "    'birkaç', 'birlikte', 'bunu', 'burada', 'biten', 'biz', \n",
+    "    'bu', 'buna', 'çünkü', 'da', 'de', 'demek', 'den', 'derken', \n",
+    "    'değil', 'daha', 'dolayı', 'edilir', 'eğer', 'en', 'fakat', \n",
+    "    'genellikle', 'gibi', 'hem', 'her', 'herhangi', 'hiç', 'ise', \n",
+    "    'işte', 'itibaren', 'iyi', 'kadar', 'karşı', 'ki', 'kime', \n",
+    "    'kısaca', 'mu', 'mü', 'nasıl', 'ne', 'neden', 'niye', 'o', \n",
+    "    'olasılıkla', 'olabilir', 'oluşur', 'önce', 'şu', 'sadece', \n",
+    "    'se', 'şey', 'şimdi', 'tabi', 'tüm', 've', 'ya', 'ya da', \n",
+    "    'yanı', 'yani', 'yılında', 'yetenekli', 'yine'\n",
     "]\n",
     "\n",
-    "#featuresların eklenmesi gerekir \n",
-    "def calculate_tfidf(documents, stop_words):\n",
+    "#features olarak top_keywordsleri belirleyerek metnin bu kelimelerin etrafında olması sağlanmalı  \n",
+    "def calculate_tfidf(combined_text, stop_words):\n",
     "    vectorizer = TfidfVectorizer(stop_words=stop_words, max_features=10000)\n",
-    "    tfidf_matrix = vectorizer.fit_transform(documents)\n",
+    "    tfidf_matrix = vectorizer.fit_transform(combined_text)\n",
     "    feature_names = vectorizer.get_feature_names_out()\n",
     "    return tfidf_matrix, feature_names\n",
     "\n",
-    "\n",
+    "#---------------------------------------------------------------------------------\n",
     "#kelimelerin ortalama skorlarını hesaplama \n",
     "def identify_low_tfidf_words(tfidf_matrix, feature_names, threshold=0.001):\n",
     "    # TF-IDF skorlarını toplayarak her kelimenin ortalama skorunu hesaplayın\n",
@@ -411,48 +881,54 @@
     "    low_tfidf_words = [feature_names[i] for i, score in enumerate(avg_scores) if score < threshold]\n",
     "    return low_tfidf_words\n",
     "\n",
-    "#kelimelerin güncellenmesi \n",
+    "#kelimelerin yeni geliştirilen eşik değere göre güncellenmesi \n",
     "def update_stop_words(existing_stop_words, low_tfidf_words):\n",
     "    updated_stop_words = set(existing_stop_words) | set(low_tfidf_words)\n",
     "    return list(updated_stop_words)\n",
     "\n",
     "\n",
-    "def iterative_update(documents, initial_stop_words, iterations=5):\n",
+    "#bu kısım detaylandırılmalı \n",
+    "def iterative_update(combined_text, initial_stop_words, iterations=5):\n",
     "    stop_words = set(initial_stop_words)\n",
     "    for _ in range(iterations):\n",
-    "        tfidf_matrix, feature_names = calculate_tfidf(documents, stop_words)\n",
+    "        tfidf_matrix, feature_names = calculate_tfidf(combined_text, stop_words)\n",
     "        low_tfidf_words = identify_low_tfidf_words(tfidf_matrix, feature_names)\n",
     "        stop_words = update_stop_words(stop_words, low_tfidf_words)\n",
     "    return list(stop_words)\n",
-    "stop_words= iterative_update\n",
+    "\n",
     "\n",
     "\n",
     "def main ():\n",
     "\n",
+    "    \n",
     "#anlam ilişkisini de kontrol edecek bir yapı oluşpturulacak title ile benzerlik kontrol ederek yüksek benzerlik içeren kelimler sıralnacak .\n",
     "\n",
     "# Dökümanları liste olarak al\n",
     "    documents_list = [doc.get('text', '') if isinstance(doc, dict) else doc for doc in list(named_documents.values())]\n",
     "\n",
+    "    #başlangıç stop değerleriyle yeni olanları arasında değişim yapma \n",
+    "    initial_stop_words = turkish_stop_words\n",
+    "    # Stop-words listesini iteratif olarak güncelleyin\n",
+    "    final_stop_words = iterative_update(documents_list, initial_stop_words)\n",
     "    #tf-ıdf hesaplama\n",
-    "    tfidf_matrix, feature_names=calculate_tfidf(documents_list,stop_words)\n",
+    "    tfidf_matrix, feature_names=calculate_tfidf(documents_list,final_stop_words)\n",
     "\n",
-    "# Veritabanından dökümanları alın\n",
-    "    named_documents, document_count = Database.get_input_documents(limit=3)\n",
+    "    \n",
     "\n",
-    "#başalngıç stop değerleriyle yeni olanları arasında değişim yapma \n",
-    "    initial_stop_words = turkish_stop_words\n",
+    "    print(\"Güncellenmiş Stop-Words Listesi:\", final_stop_words)\n",
+    "    print(\"TF-IDF Matrix Shape:\", tfidf_matrix.shape)\n",
+    "    print(\"Feature Names Sample:\", feature_names[:10])  # İlk 10 feature adını gösterir\n",
     "\n",
-    "# Stop-words listesini iteratif olarak güncelleyin\n",
-    "    final_stop_words = iterative_update(documents_list, initial_stop_words)\n",
+    "    return tfidf_matrix, feature_names,documents_list \n",
     "\n",
-    "    print(\"Güncellenmiş Stop-Words Listesi:\", final_stop_words)\n",
+    "if __name__==\"__main__\":\n",
+    "    tfidf_matrix, feature_names,documents_list= main()\n",
     "\n",
     "\n",
     "# Sonuçları yazdır\n",
-    "    print(\"İsimlendirilmiş Dökümanlar:\")\n",
-    "    for name, doc in named_documents.items():\n",
-    "        print(f\"{name}: {doc}\")\n",
+    "print(\"İsimlendirilmiş Dökümanlar:\")\n",
+    "for name, doc in named_documents.items():\n",
+    "    print(f\"{name}: {doc}\")\n",
     "\n",
     "    print(\"\\nDökümanlar Listesi:\")\n",
     "    print(documents_list)\n",
@@ -534,9 +1010,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'TfidfVectorizer' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[1], line 41\u001b[0m\n\u001b[0;32m     31\u001b[0m turkish_stop_words \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m([\n\u001b[0;32m     32\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mabide\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mabi\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mabla\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mad\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124madım\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mah\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mama\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124man\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mancak\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124maraba\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124maralar\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124maslında\u001b[39m\u001b[38;5;124m'\u001b[39m, \n\u001b[0;32m     33\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124maşşağı\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124maz\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbazı\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbelirli\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mben\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbence\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbunu\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mburada\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbiz\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbu\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbuna\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mçünkü\u001b[39m\u001b[38;5;124m'\u001b[39m, \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mönce\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mşu\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msadece\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msana\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mse\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mşey\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mşimdi\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtabi\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtüm\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mve\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mya\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mya da\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124myani\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124myine\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[0;32m     38\u001b[0m ])\n\u001b[0;32m     40\u001b[0m \u001b[38;5;66;03m# TF-IDF hesaplayıcı oluşturun ve Türkçe durak kelimelerini dahil edin\u001b[39;00m\n\u001b[1;32m---> 41\u001b[0m vectorizer \u001b[38;5;241m=\u001b[39m \u001b[43mTfidfVectorizer\u001b[49m(stop_words\u001b[38;5;241m=\u001b[39mturkish_stop_words)\n\u001b[0;32m     44\u001b[0m \u001b[38;5;124;03m\"\"\"IDF, derlemedeki belge sayısının,\u001b[39;00m\n\u001b[0;32m     45\u001b[0m \u001b[38;5;124;03mincelenen anahtar kelimeyi içeren topluluktaki belge sayısına \u001b[39;00m\n\u001b[0;32m     46\u001b[0m \u001b[38;5;124;03mbölünmesiyle elde edilen algoritmadır. \u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     49\u001b[0m \u001b[38;5;124;03mkülliyat yani incelenen tüm belgelerin adedi 10 ise ve test edilen anahtar kelime,\u001b[39;00m\n\u001b[0;32m     50\u001b[0m \u001b[38;5;124;03mkülliyattaki üç belgede görünüyorsa, bu durumda IDF değeri 0.52’dir (log (10/3)).\"\"\"\u001b[39;00m\n\u001b[0;32m     51\u001b[0m \u001b[38;5;66;03m#TF-IDF puanı; Naive Bayes ve Destek Vektör Makineleri gibi algoritmalara aktarılabilir. Böylece kelime sayısı gibi daha temel yöntemlerin sonuçları büyük ölçüde iyileştirilebilir.\u001b[39;00m\n\u001b[0;32m     52\u001b[0m \u001b[38;5;66;03m#IDF = log ( Dokuman Sayısı / Terimin Geçtiği Dokuman Sayısı )\u001b[39;00m\n\u001b[0;32m     53\u001b[0m \u001b[38;5;66;03m#dokuman sayısılarını almakla başlayacağız.\u001b[39;00m\n\u001b[0;32m     54\u001b[0m \u001b[38;5;66;03m#  : titlelerın sayısı / terimler ise \u001b[39;00m\n",
+      "\u001b[1;31mNameError\u001b[0m: name 'TfidfVectorizer' is not defined"
+     ]
+    }
+   ],
    "source": [
     "\n",
     "#---------------------------------------------------------------------------------------------------------------------------------\n",