Upload 41 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+img/graph-tool.png filter=lfs diff=lfs merge=lfs -text
+img/zhu.png filter=lfs diff=lfs merge=lfs -text

Config/config.py

@@ -0,0 +1,14 @@
+# 向量数据库选择 【chroma：1】 ，【faiss 2】,【ElasticsearchStore 3】
+VECTOR_DB = 2
+DB_directory = "./Chroma_db/"
+if VECTOR_DB==2:
+    DB_directory ="./Faiss_db/"
+elif VECTOR_DB==3:
+    DB_directory = "es"
+
+# 配置neo4j
+neo4j_host = "bolt://localhost:7687"
+neo4j_name = "neo4j"
+neo4j_pwd = "12345678"
+# 测试了 llama3：8b,gemma2:9b,qwen2:7b,glm4:9b，arcee-ai/arcee-agent:latest  目前来看 qwen2:7 效果最好
+neo4j_model = "qwen2:7b"

Faiss_db/sss1/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bb588f4bd46218f42b045c42163bdcf3cc76a19e37458823ceaeaf8a1454e3b
+size 9362

Neo4j/graph_extract.py

@@ -0,0 +1,69 @@
+from langchain_community.llms import Ollama
+from Config.config import neo4j_model
+
+# 测试了 llama3：8b,gemma2:9b,qwen2:7b,glm4:9b，arcee-ai/arcee-agent:latest  目前来看 qwen2:7 效果最好
+llm = Ollama(model=neo4j_model)
+
+json_example = {'edges': [
+    {
+        'label': 'label 1',
+        'source': 'source 1',
+        'target': 'target 1'},
+    {
+        'label': 'label 1',
+        'source': 'source 1',
+        'target': 'target 1'}
+],
+    'nodes': [{'name': 'label 1'},
+              {'name': 'label 2'},
+              {'name': 'label 3'}]
+}
+
+__retriever_prompt = f"""
+            您是一名专门从事知识图谱创建的人工智能专家，目标是根据给定的输入或请求捕获关系。
+            基于各种形式的用户输入，如段落、电子邮件、文本文件等。
+            你的任务是根据输入创建一个知识图谱。
+            nodes中每个元素只有一个name参数，name对应的值是一个实体，实体来自输入的词语或短语。
+             edges还必须有一个label参数，其中label是输入中的直接词语或短语,edges中的source和target取自nodes中的name。
+
+            仅使用JSON进行响应，其格式可以在python中进行jsonify，并直接输入cy.add（data），
+            您可以参考给定的示例：{json_example}。存储node和edge的数组中，最后一个元素后边不要有逗号，
+            确保边的目标和源与现有节点匹配。
+            不要在JSON的上方和下方包含markdown三引号，直接用花括号括起来。
+            """
+
+
+def generate_graph_info(raw_text: str) -> str | None:
+    """
+    generate graph info from raw text
+    :param raw_text:
+    :return:
+    """
+    messages = [
+        {"role": "system", "content": "你现在扮演信息抽取的角色，要求根据用户输入和AI的回答，正确提取出信息,记得不多对实体进行翻译。"},
+        {"role": "user", "content": raw_text},
+        {"role": "user", "content": __retriever_prompt}
+    ]
+    print("解析中....")
+    for i in range(3):
+        graph_info_result = llm.invoke(messages)
+        if len(graph_info_result) < 10:
+            print("-------", i, "-------------------")
+            continue
+        else:
+            break
+    print(graph_info_result)
+    return graph_info_result
+
+
+def update_graph(raw_text):
+    #     raw_text = request.json.get('text', '')
+    try:
+        result = generate_graph_info(raw_text)
+        if '```' in result:
+            graph_data = eval(result.split('```', 2)[1].replace("json", ''))
+        else:
+            graph_data = eval(str(result))
+        return graph_data
+    except Exception as e:
+        return {'error': f"Error parsing graph data: {str(e)}"}

Neo4j/neo4j_op.py

@@ -0,0 +1,105 @@
+from py2neo import Graph, Node, Relationship
+from langchain_community.document_loaders import TextLoader,UnstructuredCSVLoader, UnstructuredPDFLoader,UnstructuredWordDocumentLoader,UnstructuredExcelLoader,UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+
+
+class KnowledgeGraph:
+    def __init__(self, uri, user, password):
+        self.graph = Graph(uri, auth=(user, password))
+
+    def parse_data(self,file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file,encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        return data
+
+    # 切分 数据
+    def split_files(self, files,chunk_size=500, chunk_overlap=100):
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = text_splitter.split_documents(tmps)
+
+        return splits
+
+    def create_node(self, label, properties):
+        matcher = self.graph.nodes.match(label, **properties)
+        if matcher.first():
+            return matcher.first()
+        else:
+            node = Node(label, **properties)
+            self.graph.create(node)
+            return node
+
+    def create_relationship(self, label1, properties1, label2, properties2, relationship_type,
+                            relationship_properties={}):
+        node1 = self.create_node(label1, properties1)
+        node2 = self.create_node(label2, properties2)
+
+        matcher = self.graph.match((node1, node2), r_type=relationship_type)
+        for rel in matcher:
+            if all(rel[key] == value for key, value in relationship_properties.items()):
+                return rel
+
+        relationship = Relationship(node1, relationship_type, node2, **relationship_properties)
+        self.graph.create(relationship)
+        return relationship
+
+    def delete_node(self, label, properties):
+        matcher = self.graph.nodes.match(label, **properties)
+        node = matcher.first()
+        if node:
+            self.graph.delete(node)
+            return True
+        return False
+
+    def update_node(self, label, identifier, updates):
+        matcher = self.graph.nodes.match(label, **identifier)
+        node = matcher.first()
+        if node:
+            for key, value in updates.items():
+                node[key] = value
+            self.graph.push(node)
+            return node
+        return None
+
+    def find_node(self, label, properties):
+        matcher = self.graph.nodes.match(label, **properties)
+        return list(matcher)
+
+    def create_nodes(self, label, properties_list):
+        nodes = []
+        for properties in properties_list:
+            node = self.create_node(label, properties)
+            nodes.append(node)
+        return nodes
+
+    def create_relationships(self, relationships):
+        created_relationships = []
+        for rel in relationships:
+            label1, properties1, label2, properties2, relationship_type = rel
+            relationship = self.create_relationship(label1, properties1, label2, properties2, relationship_type)
+            created_relationships.append(relationship)
+        return created_relationships
+

Ollama_api/ollama_api.py

@@ -0,0 +1,21 @@
+import requests
+import json
+
+# 提供api获取ollama 模型列表
+def get_llm():
+    respone = requests.get(url="http://localhost:11434/api/tags")
+    result = json.loads(respone.content)
+    llms = []
+    for llm in result["models"]:
+        if "code" not in llm["name"] and "embed" not in llm["name"]:
+            llms.append(llm["name"])
+    return llms
+
+def get_embeding_model():
+    respone = requests.get(url="http://localhost:11434/api/tags")
+    result = json.loads(respone.content)
+    llms = []
+    for llm in result["models"]:
+        if "embed" in llm["name"]:
+            llms.append(llm["name"])
+    return llms

app.py

@@ -0,0 +1,354 @@
+import gradio as gr
+import threading
+import asyncio
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
+import requests
+import json
+
+# 假设这些是您的自定义模块，需要根据实际情况进行调整
+from Config.config import VECTOR_DB, DB_directory
+from Ollama_api.ollama_api import *
+from rag.rag_class import *
+
+# 设置日志
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# 根据VECTOR_DB选择合适的向量数据库
+if VECTOR_DB == 1:
+    from embeding.chromadb import ChromaDB as vectorDB
+    vectordb = vectorDB(persist_directory=DB_directory)
+elif VECTOR_DB == 2:
+    from embeding.faissdb import FaissDB as vectorDB
+    vectordb = vectorDB(persist_directory=DB_directory)
+elif VECTOR_DB == 3:
+    from embeding.elasticsearchStore import ElsStore as vectorDB
+    vectordb = vectorDB()
+
+# 存储上传的文件
+uploaded_files = []
+
+@lru_cache(maxsize=100)
+def get_knowledge_base_files():
+    cl_dict = {}
+    cols = vectordb.get_all_collections_name()
+    for c_name in cols:
+        cl_dict[c_name] = vectordb.get_collcetion_content_files(c_name)
+    return cl_dict
+
+knowledge_base_files = get_knowledge_base_files()
+
+def upload_files(files):
+    if files:
+        new_files = [file.name for file in files]
+        uploaded_files.extend(new_files)
+        update_knowledge_base_files()
+        logger.info(f"Uploaded files: {new_files}")
+        return update_file_list(), new_files, "<div style='color: green; padding: 10px; border: 2px solid green; border-radius: 5px;'>Upload successful!</div>"
+    update_knowledge_base_files()
+    return update_file_list(), [], "<div style='color: red; padding: 10px; border: 2px solid red; border-radius: 5px;'>Upload failed!</div>"
+
+def delete_files(selected_files):
+    global uploaded_files
+    uploaded_files = [f for f in uploaded_files if f not in selected_files]
+    if selected_files:
+        update_knowledge_base_files()
+        logger.info(f"Deleted files: {selected_files}")
+        return update_file_list(), "<div style='color: green; padding: 10px; border: 2px solid green; border-radius: 5px;'>Delete successful!</div>"
+    update_knowledge_base_files()
+    return update_file_list(), "<div style='color: red; padding: 10px; border: 2px solid red; border-radius: 5px;'>Delete failed!</div>"
+
+def delete_collection(selected_knowledge_base):
+    if selected_knowledge_base and selected_knowledge_base != "创建知识库":
+        vectordb.delete_collection(selected_knowledge_base)
+        update_knowledge_base_files()
+        logger.info(f"Deleted collection: {selected_knowledge_base}")
+        return update_knowledge_base_dropdown(), "<div style='color: green; padding: 10px; border: 2px solid green; border-radius: 5px;'>Collection deleted successfully!</div>"
+    return update_knowledge_base_dropdown(), "<div style='color: red; padding: 10px; border: 2px solid red; border-radius: 5px;'>Delete collection failed!</div>"
+
+async def async_vectorize_files(selected_files, selected_knowledge_base, new_kb_name, chunk_size, chunk_overlap):
+    if selected_files:
+        if selected_knowledge_base == "创建知识库":
+            knowledge_base = new_kb_name
+            vectordb.create_collection(selected_files, knowledge_base, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        else:
+            knowledge_base = selected_knowledge_base
+            vectordb.add_chroma(selected_files, knowledge_base, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+        if knowledge_base not in knowledge_base_files:
+            knowledge_base_files[knowledge_base] = []
+        knowledge_base_files[knowledge_base].extend(selected_files)
+
+        logger.info(f"Vectorized files: {selected_files} for knowledge base: {knowledge_base}")
+        await asyncio.sleep(0)  # 允许其他任务执行
+        return f"Vectorized files: {', '.join(selected_files)}\nKnowledge Base: {knowledge_base}\nUploaded Files: {', '.join(uploaded_files)}", "<div style='color: green; padding: 10px; border: 2px solid green; border-radius: 5px;'>Vectorization successful!</div>"
+    return "", "<div style='color: red; padding: 10px; border: 2px solid red; border-radius: 5px;'>Vectorization failed!</div>"
+
+def update_file_list():
+    return gr.update(choices=uploaded_files, value=[])
+
+def search_knowledge_base(selected_knowledge_base):
+    if selected_knowledge_base in knowledge_base_files:
+        kb_files = knowledge_base_files[selected_knowledge_base]
+        return gr.update(choices=kb_files, value=[])
+    return gr.update(choices=[], value=[])
+
+def update_knowledge_base_files():
+    global knowledge_base_files
+    knowledge_base_files = get_knowledge_base_files()
+
+# 处理聊天消息的函数
+chat_history = []
+
+def safe_chat_response(model_dropdown, vector_dropdown, chat_knowledge_base_dropdown, chain_dropdown, message):
+    try:
+        return chat_response(model_dropdown, vector_dropdown, chat_knowledge_base_dropdown, chain_dropdown, message)
+    except Exception as e:
+        logger.error(f"Error in chat response: {str(e)}")
+        return f"<div style='color: red;'>Error: {str(e)}</div>", ""
+
+def chat_response(model_dropdown, vector_dropdown, chat_knowledge_base_dropdown, chain_dropdown, message):
+    global chat_history
+    if message:
+        chat_history.append(("User", message))
+        if chat_knowledge_base_dropdown == "仅使用模型":
+            rag = RAG_class(model=model_dropdown,persist_directory=DB_directory)
+            answer = rag.mult_chat(chat_history)
+        if chat_knowledge_base_dropdown and chat_knowledge_base_dropdown != "仅使用模型":
+            rag = RAG_class(model=model_dropdown, embed=vector_dropdown, c_name=chat_knowledge_base_dropdown, persist_directory=DB_directory)
+            if chain_dropdown == "复杂召回方式":
+                questions = rag.decomposition_chain(message)
+                answer = rag.rag_chain(questions)
+            elif chain_dropdown == "简单召回方式":
+                answer = rag.simple_chain(message)
+            else:
+                answer = rag.rerank_chain(message)
+
+        response = f" {answer}"
+        chat_history.append(("Bot", response))
+    return format_chat_history(chat_history), ""
+
+def clear_chat():
+    global chat_history
+    chat_history = []
+    return format_chat_history(chat_history)
+
+def format_chat_history(history):
+    formatted_history = ""
+    for user, msg in history:
+        if user == "User":
+            formatted_history += f'''
+            <div style="text-align: right; margin: 10px;">
+                <div style="display: inline-block; background-color: #DCF8C6; padding: 10px; border-radius: 10px; max-width: 60%;">
+                    {msg}
+                </div>
+                <b>:User</b>
+            </div>
+            '''
+        else:
+            if "```" in msg:  # 检测是否包含代码片段
+                code_content = msg.split("```")[1]
+                formatted_history += f'''
+                <div style="text-align: left; margin: 10px;">
+                    <b>Bot:</b>
+                    <div style="display: inline-block; background-color: #F1F0F0; padding: 10px; border-radius: 10px; max-width: 60%;">
+                        <pre><code>{code_content}</code></pre>
+                    </div>
+                </div>
+                '''
+            else:
+                formatted_history += f'''
+                <div style="text-align: left; margin: 10px;">
+                    <b>Bot:</b>
+                    <div style="display: inline-block; background-color: #F1F0F0; padding: 10px; border-radius: 10px; max-width: 60%;">
+                        {msg}
+                    </div>
+                </div>
+                '''
+    return formatted_history
+
+def clear_status():
+    upload_status.update("")
+    delete_status.update("")
+    vectorize_status.update("")
+    delete_collection_status.update("")
+
+def handle_knowledge_base_selection(selected_knowledge_base):
+    if selected_knowledge_base == "创建知识库":
+        return gr.update(visible=True, interactive=True), gr.update(choices=[], value=[]), gr.update(visible=False)
+    elif selected_knowledge_base == "仅使用模型":
+        return gr.update(visible=False, interactive=False), gr.update(choices=[], value=[]), gr.update(visible=False)
+    else:
+        return gr.update(visible=False, interactive=False), search_knowledge_base(selected_knowledge_base), gr.update(visible=True)
+
+def update_knowledge_base_dropdown():
+    global knowledge_base_files
+    choices = ["创建知识库"] + list(knowledge_base_files.keys())
+    return gr.update(choices=choices)
+
+def update_chat_knowledge_base_dropdown():
+    global knowledge_base_files
+    choices = ["仅使用模型"] + list(knowledge_base_files.keys())
+    return gr.update(choices=choices)
+
+
+# SearxNG搜索函数
+def search_searxng(query):
+    searxng_url = 'http://localhost:8080/search'  # 替换为你的SearxNG实例URL
+    params = {
+        'q': query,
+        'format': 'json'
+    }
+    response = requests.get(searxng_url, params=params)
+    response.raise_for_status()
+    return response.json()
+
+
+# Ollama总结函数
+def summarize_with_ollama(model_dropdown,text, question):
+    prompt = """
+        根据下边的内容，回答用户问题，
+        内容为：‘{0}‘\n
+        问题为：{1}
+    """.format(text, question)
+    ollama_url = 'http://localhost:11434/api/generate'  # 替换为你的Ollama实例URL
+    data = {
+        'model': model_dropdown,
+        "prompt": prompt,
+        "stream": False
+    }
+    response = requests.post(ollama_url, json=data)
+    response.raise_for_status()
+    return response.json()
+
+
+# 处理函数
+def ai_web_search(model_dropdown,user_query):
+    # 使用SearxNG进行搜索
+    search_results = search_searxng(user_query)
+    search_texts = [result['title'] + "\n" + result['content'] for result in search_results['results']]
+    combined_text = "\n\n".join(search_texts)
+
+    # 使用Ollama进行总结
+    summary = summarize_with_ollama(model_dropdown,combined_text, user_query)
+    # print(summary)
+    # 返回结果
+    return summary['response']
+# 添加新的函数来处理AI网络搜索
+# def ai_web_search(model_dropdown, query):
+#     try:
+#         # 这里添加实际的网络搜索和AI处理逻辑
+#         # 这只是一个示例，您需要根据实际情况实现
+#         search_result = f"搜索结果: {query}"
+#         ai_response = f"AI回答: 基于搜索结果，对于'{query}'的回答是..."
+#         return f"{search_result}\n\n{ai_response}"
+#     except Exception as e:
+#         logger.error(f"Error in AI web search: {str(e)}")
+#         return f"<div style='color: red;'>Error: {str(e)}</div>"
+
+# 创建 Gradio 界面
+with gr.Blocks() as demo:
+    with gr.Column():
+        # 添加标题
+        title = gr.HTML("<h1 style='text-align: center; font-size: 32px; font-weight: bold;'>RAG精致系统</h1>")
+        # 添加公告栏
+        announcement = gr.HTML("<div style='text-align: center; font-size: 18px; color: red;'>公告栏: RAG精致系统，【检索增强生成】系统！<br/>莫大大</div>")
+
+        with gr.Tabs():
+            with gr.TabItem("知识库"):
+                knowledge_base_dropdown = gr.Dropdown(choices=["创建知识库"] + list(knowledge_base_files.keys()),
+                                                      label="选择知识库")
+                new_kb_input = gr.Textbox(label="输入新的知识库名称", visible=False, interactive=True)
+                file_input = gr.Files(label="Upload files")
+                upload_btn = gr.Button("Upload")
+                file_list = gr.CheckboxGroup(label="Uploaded Files")
+                delete_btn = gr.Button("Delete Selected Files")
+                with gr.Row():
+                    chunk_size_dropdown = gr.Dropdown(choices=[50, 100, 200, 300, 500, 700], label="chunk_size", value=200)
+                    chunk_overlap_dropdown = gr.Dropdown(choices=[20, 50, 100, 200], label="chunk_overlap", value=50)
+                    vectorize_btn = gr.Button("Vectorize Selected Files")
+                delete_collection_btn = gr.Button("Delete Collection")
+                upload_status = gr.HTML()
+                delete_status = gr.HTML()
+                vectorize_status = gr.HTML()
+                delete_collection_status = gr.HTML()
+
+            with gr.TabItem("Chat"):
+                with gr.Row():
+                    model_dropdown = gr.Dropdown(choices=get_llm(), label="模型")
+                    vector_dropdown = gr.Dropdown(choices=get_embeding_model(), label="向量")
+                    chat_knowledge_base_dropdown = gr.Dropdown(choices=["仅使用模型"] + vectordb.get_all_collections_name(), label="知识库")
+                    chain_dropdown = gr.Dropdown(choices=["复杂召回方式", "简单召回方式","rerank"], label="chain方式", visible=False)
+                chat_display = gr.HTML(label="Chat History")
+                chat_input = gr.Textbox(label="Type a message")
+                chat_btn = gr.Button("Send")
+                clear_btn = gr.Button("Clear Chat History")
+
+            with gr.TabItem("AI网络搜索"):
+                with gr.Row():
+                    web_search_model_dropdown = gr.Dropdown(choices=get_llm(), label="模型")
+                web_search_output = gr.Textbox(label="搜索结果和AI回答", lines=10)
+                web_search_input = gr.Textbox(label="输入搜索查询")
+
+                web_search_btn = gr.Button("搜索")
+
+    def handle_upload(files):
+        upload_result, new_files, status = upload_files(files)
+        threading.Thread(target=clear_status).start()
+        return upload_result, new_files, status, update_chat_knowledge_base_dropdown()
+
+    def handle_delete(selected_knowledge_base, selected_files):
+        tmp = []
+        cols_files_tmp = vectordb.get_collcetion_content_files(c_name=selected_knowledge_base)
+        for i in selected_files:
+            if i in cols_files_tmp:
+                tmp.append(i)
+        del cols_files_tmp
+        if tmp:
+            vectordb.del_files(tmp, c_name=selected_knowledge_base)
+        del tmp
+        delete_result, status = delete_files(selected_files)
+        threading.Thread(target=clear_status).start()
+        return delete_result, status, update_chat_knowledge_base_dropdown()
+
+    def handle_vectorize(selected_files, selected_knowledge_base, new_kb_name, chunk_size, chunk_overlap):
+        vectorize_result, status = asyncio.run(async_vectorize_files(selected_files, selected_knowledge_base, new_kb_name, chunk_size, chunk_overlap))
+        threading.Thread(target=clear_status).start()
+        return vectorize_result, status, update_knowledge_base_dropdown(), update_chat_knowledge_base_dropdown()
+
+    def handle_delete_collection(selected_knowledge_base):
+        result, status = delete_collection(selected_knowledge_base)
+        threading.Thread(target=clear_status).start()
+        return result, status, update_chat_knowledge_base_dropdown()
+
+    knowledge_base_dropdown.change(
+        handle_knowledge_base_selection,
+        inputs=knowledge_base_dropdown,
+        outputs=[new_kb_input, file_list, chain_dropdown]
+    )
+    upload_btn.click(handle_upload, inputs=file_input, outputs=[file_list, file_list, upload_status, chat_knowledge_base_dropdown])
+    delete_btn.click(handle_delete, inputs=[knowledge_base_dropdown, file_list], outputs=[file_list, delete_status, chat_knowledge_base_dropdown])
+    vectorize_btn.click(handle_vectorize, inputs=[file_list, knowledge_base_dropdown, new_kb_input, chunk_size_dropdown, chunk_overlap_dropdown],
+                        outputs=[gr.Textbox(visible=False), vectorize_status, knowledge_base_dropdown, chat_knowledge_base_dropdown])
+    delete_collection_btn.click(handle_delete_collection, inputs=knowledge_base_dropdown,
+                                outputs=[knowledge_base_dropdown, delete_collection_status, chat_knowledge_base_dropdown])
+
+    chat_btn.click(chat_response, inputs=[model_dropdown, vector_dropdown, chat_knowledge_base_dropdown, chain_dropdown, chat_input], outputs=[chat_display, chat_input])
+    clear_btn.click(clear_chat, outputs=chat_display)
+
+    chat_knowledge_base_dropdown.change(
+        fn=lambda selected: gr.update(visible=selected != "仅使用模型"),
+        inputs=chat_knowledge_base_dropdown,
+        outputs=chain_dropdown
+    )
+
+    # 添加新的点击事件处理
+    web_search_btn.click(
+        ai_web_search,
+        inputs=[web_search_model_dropdown, web_search_input],
+        outputs=web_search_output
+    )
+
+demo.launch(debug=True,share=True)

embeding/asr_utils.py

@@ -0,0 +1,17 @@
+#coding:utf-8
+from funasr import AutoModel
+# paraformer-zh is a multi-functional asr model
+# use vad, punc, spk or not as you need
+model = AutoModel(model="paraformer-zh",  vad_model="fsmn-vad", punc_model="ct-punc",
+                  # spk_model="cam++"
+                  )
+def get_spk_txt(file):
+    res = model.generate(input=file,
+                batch_size_s=300,
+                hotword='魔搭')
+    print(res[0]["text"])
+    fw = "embeding/tmp.txt"
+    f = open(fw,"w",encoding="utf-8")
+    f.write('"context"\n'+res[0]["text"])
+    f.close()
+    return fw

embeding/chromadb.py

@@ -0,0 +1,134 @@
+from langchain_community.vectorstores import Chroma
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.document_loaders import TextLoader,UnstructuredCSVLoader, UnstructuredPDFLoader,UnstructuredWordDocumentLoader,UnstructuredExcelLoader,UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from .asr_utils import get_spk_txt
+
+class ChromaDB():
+    def __init__(self, embedding="mofanke/acge_text_embedding:latest", persist_directory="./Chroma_db/"):
+
+        self.embedding = OllamaEmbeddings(model=embedding)
+        self.persist_directory = persist_directory
+        self.chromadb = Chroma(persist_directory=persist_directory)
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=50)
+
+    def parse_data(self,file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file,encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        if "mp3" in file.lower() or "mp4" in file.lower() or "wav" in file.lower():
+            # 语音解析成文字
+            fw = get_spk_txt(file)
+            loaders = UnstructuredCSVLoader(fw)
+            data = loaders.load()
+            tmp = []
+            for i in data:
+                i.metadata["source"] = file
+                tmp.append(i)
+            data = tmp
+        return data
+
+    # 创建 新的collection 并且初始化
+    def create_collection(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = self.chromadb.from_documents(documents=splits, collection_name=c_name,
+                                                   embedding=self.embedding, persist_directory=self.persist_directory)
+        print("数据块总量:", vectorstore._collection.count())
+
+        return vectorstore
+
+    # 添加 数据到已有数据库
+    def add_chroma(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始添加文件...")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = Chroma(persist_directory=self.persist_directory, collection_name=c_name,
+                             embedding_function=self.embedding)
+        vectorstore.add_documents(splits)
+        print("数据块总量:", vectorstore._collection.count())
+
+        return vectorstore
+
+    # 删除 某个collection中的 某个文件
+    def del_files(self, del_files_name, c_name):
+
+        vectorstore = self.chromadb._client.get_collection(c_name)
+        del_ids = []
+        vec_dict = vectorstore.get()
+        for id, md in zip(vec_dict["ids"], vec_dict["metadatas"]):
+            for dl in del_files_name:
+                if dl in md["source"]:
+                    del_ids.append(id)
+        vectorstore.delete(ids=del_ids)
+        print("数据块总量:", vectorstore.count())
+
+        return vectorstore
+
+    # 删除某个 知识库 collection
+    def delete_collection(self, c_name):
+
+        self.chromadb._client.delete_collection(c_name)
+
+    # 获取目前所有 collection
+    def get_all_collections_name(self):
+        cl_names = []
+
+        test = self.chromadb._client.list_collections()
+        for i in range(len(test)):
+            cl_names.append(test[i].name)
+        return cl_names
+
+    # 获取 collection中的所有文件
+    def get_collcetion_content_files(self, c_name):
+        vectorstore = self.chromadb._client.get_collection(c_name)
+        c_files = []
+        vec_dict = vectorstore.get()
+        for md in vec_dict["metadatas"]:
+            c_files.append(md["source"])
+        return list(set(c_files))
+
+
+# if __name__ == "__main__":
+#     chromadb = ChromaDB()
+#     c_name = "sss3"
+#
+#     print(chromadb.get_all_collections_name())
+#     chromadb.create_collection(["data/���内科学.txt", "data/jl.pdf"], c_name=c_name)
+#     print(chromadb.get_all_collections_name())
+#     chromadb.add_chroma(["data/儿科学.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     chromadb.del_files(["data/肾内科学.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     print(chromadb.get_all_collections_name())
+#     chromadb.delete_collection(c_name=c_name)
+#     print(chromadb.get_all_collections_name())

embeding/elasticsearchStore.py

@@ -0,0 +1,147 @@
+from elasticsearch import Elasticsearch
+from langchain_elasticsearch.vectorstores import ElasticsearchStore
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.document_loaders import TextLoader, UnstructuredCSVLoader, UnstructuredPDFLoader, \
+    UnstructuredWordDocumentLoader, UnstructuredExcelLoader, UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from .asr_utils import get_spk_txt
+import requests
+
+
+class ElsStore():
+    def __init__(self, embedding="mofanke/acge_text_embedding:latest", es_url="http://localhost:9200",
+                 index_name='test_index'):
+        self.embedding = OllamaEmbeddings(model=embedding)
+        self.es_url = es_url
+        self.elastic_vector_search = ElasticsearchStore(
+            es_url=self.es_url,
+            index_name=index_name,
+            embedding=self.embedding
+        )
+
+    def parse_data(self, file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file, encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        if "mp3" in file.lower() or "mp4" in file.lower() or "wav" in file.lower():
+            # 语音解析成文字
+            fw = get_spk_txt(file)
+            loaders = UnstructuredCSVLoader(fw)
+            data = loaders.load()
+            tmp = []
+            for i in data:
+                i.metadata["source"] = file
+                tmp.append(i)
+            data = tmp
+        return data
+
+    def get_count(self, c_name):
+        # 获取index-anme中的数据块数
+
+        # 初始化 Elasticsearch 客户端
+        es = Elasticsearch([{
+            'host': self.es_url.split(":")[1][2:],
+            'port': int(self.es_url.split(":")[2]),
+            'scheme': 'http'  # 指定使用的协议
+        }])
+
+        # 指定索引名称
+        index_name = c_name
+
+        # 获取文档总数
+        response = es.count(index=index_name)
+
+        # 输出文档总数
+        return response['count']
+
+    # 创建 新的index_name 并且初始化
+    def create_collection(self, files, c_name, chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        self.elastic_vector_search = ElasticsearchStore.from_documents(
+            documents=splits,
+            embedding=self.embedding,
+            es_url=self.es_url,
+            index_name=c_name,
+        )
+
+        self.elastic_vector_search.client.indices.refresh(index=c_name)
+
+        print("数据块总量:", self.get_count(c_name))
+
+        return self.elastic_vector_search
+
+    # 添加 数据到已有数据库
+    def add_chroma(self, files, c_name, chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始添加文件...")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        self.elastic_vector_search = ElasticsearchStore(
+            es_url=self.es_url,
+            index_name=c_name,
+            embedding=self.embedding
+        )
+        self.elastic_vector_search.add_documents(splits)
+        self.elastic_vector_search.client.indices.refresh(index=c_name)
+        print("数据块总量:", self.get_count(c_name))
+
+        return self.elastic_vector_search
+
+    # 删除某个 知识库 collection
+    def delete_collection(self, c_name):
+        url = self.es_url + "/" + c_name
+        # 发送 DELETE 请求
+        response = requests.delete(url)
+
+        # 检查响应状态码
+        if response.status_code == 200:
+            return f"索引 'test-basic1' 已成功删除。"
+        elif response.status_code == 404:
+            return f"索引 'test-basic1' 不存在。"
+        else:
+            return f"删除索引时出错: {response.status_code}, {response.text}"
+
+    # 获取目前所有 index_names
+    def get_all_collections_name(self):
+        indices = self.elastic_vector_search.client.indices.get_alias()
+        index_names = list(indices.keys())
+
+        return index_names
+
+    def get_collcetion_content_files(self,c_name):
+        return []
+
+    # 删除 某个collection中的 某个文件
+    def del_files(self, del_files_name, c_name):
+        return None
+
+

embeding/faissdb.py

@@ -0,0 +1,138 @@
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.document_loaders import TextLoader,UnstructuredCSVLoader, UnstructuredPDFLoader,UnstructuredWordDocumentLoader,UnstructuredExcelLoader,UnstructuredMarkdownLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import shutil
+import os
+from .asr_utils import get_spk_txt
+
+
+class FaissDB():
+    def __init__(self, embedding="mofanke/acge_text_embedding:latest", persist_directory="./Faiss_db/"):
+
+        self.embedding = OllamaEmbeddings(model=embedding)
+        self.persist_directory = persist_directory
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=300, chunk_overlap=50, add_start_index=True)
+
+    def parse_data(self,file):
+        if "txt" in file.lower() or "csv" in file.lower():
+            try:
+                loaders = UnstructuredCSVLoader(file)
+                data = loaders.load()
+            except:
+                loaders = TextLoader(file,encoding="utf-8")
+                data = loaders.load()
+        if ".doc" in file.lower() or ".docx" in file.lower():
+            loaders = UnstructuredWordDocumentLoader(file)
+            data = loaders.load()
+        if "pdf" in file.lower():
+            loaders = UnstructuredPDFLoader(file)
+            data = loaders.load()
+        if ".xlsx" in file.lower():
+            loaders = UnstructuredExcelLoader(file)
+            data = loaders.load()
+        if ".md" in file.lower():
+            loaders = UnstructuredMarkdownLoader(file)
+            data = loaders.load()
+        if "mp3" in file.lower() or "mp4" in file.lower() or "wav" in file.lower():
+            # 语音解析成文字
+            fw = get_spk_txt(file)
+            loaders = UnstructuredCSVLoader(fw)
+            data = loaders.load()
+            tmp = []
+            for i in data:
+                i.metadata["source"] = file
+                tmp.append(i)
+            data = tmp
+        return data
+
+    # 创建 新的collection 并且初始化
+    def create_collection(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始创建数据库 ....")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = FAISS.from_documents(documents=splits,
+                                           embedding=self.embedding)
+        vectorstore.save_local(self.persist_directory + c_name)
+        print("数据块总量:", vectorstore.index.ntotal)
+
+        return vectorstore
+
+    # 添加 数据到已有数据库
+    def add_chroma(self, files, c_name,chunk_size=200, chunk_overlap=50):
+        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        print("开始添加文件...")
+        tmps = []
+        for file in files:
+            data = self.parse_data(file)
+            tmps.extend(data)
+
+        splits = self.text_splitter.split_documents(tmps)
+
+        vectorstore = FAISS.load_local(folder_path=self.persist_directory + c_name, embeddings=self.embedding,
+                                       allow_dangerous_deserialization=True)
+        vectorstore.add_documents(documents=splits)
+        vectorstore.save_local("Faiss_db/" + c_name)
+        print("数据块总量:", vectorstore.index.ntotal)
+
+        return vectorstore
+
+    # 删除 某个collection中的 某个文件
+    def del_files(self, del_files_name, c_name):
+
+        vectorstore = FAISS.load_local(folder_path=self.persist_directory + c_name, embeddings=self.embedding,
+                                       allow_dangerous_deserialization=True)
+        del_ids = []
+        vec_dict = vectorstore.docstore._dict
+        for id, md in vec_dict.items():
+            for dl in del_files_name:
+                if dl in md.metadata["source"]:
+                    del_ids.append(id)
+        vectorstore.delete(ids=del_ids)
+        vectorstore.save_local(self.persist_directory + c_name)
+        print("数据块总量:", vectorstore.index.ntotal)
+
+        return vectorstore
+
+    # 删除某个 知识库 collection
+    def delete_collection(self, c_name):
+        shutil.rmtree(self.persist_directory + c_name)
+
+    # 获取目前所有 collection
+    def get_all_collections_name(self):
+        cl_names = [i for i in os.listdir(self.persist_directory) if os.path.isdir(self.persist_directory+i)]
+
+        return cl_names
+
+    # 获取 collection中的所有文件
+    def get_collcetion_content_files(self, c_name):
+        vectorstore = FAISS.load_local(folder_path=self.persist_directory + c_name, embeddings=self.embedding,
+                                       allow_dangerous_deserialization=True)
+        c_files = []
+        vec_dict = vectorstore.docstore._dict
+        for _, md in vec_dict.items():
+            c_files.append(md.metadata["source"])
+
+        return list(set(c_files))
+
+
+# if __name__ == "__main__":
+#     chromadb = FaissDB()
+#     c_name = "sss3"
+#
+#     print(chromadb.get_all_collections_name())
+#     chromadb.create_collection(["data/jl.txt", "data/jl.pdf"], c_name=c_name)
+#     print(chromadb.get_all_collections_name())
+#     chromadb.add_chroma(["data/tmp.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     chromadb.del_files(["data/tmp.txt"], c_name=c_name)
+#     print(c_name, "包含的文件:", chromadb.get_collcetion_content_files(c_name))
+#     print(chromadb.get_all_collections_name())
+#     chromadb.delete_collection(c_name=c_name)
+#     print(chromadb.get_all_collections_name())

embeding/tmp.txt

@@ -0,0 +1,2 @@
+"context"
+你是不是觉得自己说话的声音直来直去呢？现在告诉你一个主持人吐字的小秘密，那就是每个字在口腔当中像是翻跟头一样打一圈再出来。比如说故人西辞黄鹤楼，而不是故人西辞黄鹤楼。再比如说乌衣巷口夕阳斜，而不是乌衣巷口夕阳斜，你也试试看抖音。

graph_demo_ui.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+from flask import Flask, render_template, request, jsonify
+import json
+from dotenv import load_dotenv
+from langchain_community.llms import Ollama
+
+
+load_dotenv()
+
+app = Flask(__name__)
+
+# 测试了 llama3：8b,gemma2:9b,qwen2:7b,glm4:9b，arcee-ai/arcee-agent:latest  目前来看 qwen2:7 效果最好
+llm = Ollama(model="qwen2:7b")
+
+
+json_example = {'edges': [{'data': {'color': '#FFA07A',
+                                    'label': 'label 1',
+                                    'source': 'source 1',
+                                    'target': 'target 1'}},
+                          {'data': {'color': '#FFA07A',
+                                    'label': 'label 2',
+                                    'source': 'source 2',
+                                    'target': 'target 2'}}
+                          ],
+                'nodes': [{'data': {'color': '#FFC0CB', 'id': 'id 1', 'label': 'label 1'}},
+                          {'data': {'color': '#90EE90', 'id': 'id 2', 'label': 'label 2'}},
+                          {'data': {'color': '#87CEEB', 'id': 'id 3', 'label': 'label 3'}}]}
+
+
+
+__retriever_prompt = f"""
+            您是一名专门从事知识图谱创建的人工智能专家，目标是根据给定的输入或请求捕获关系。
+            基于各种形式的用户输入，如段落、电子邮件、文本文件等。
+            你的任务是根据输入创建一个知识图谱。
+            nodes必须具有label参数，并且label是来自输入的词语或短语，nodes必须具有id参数,id的格式是"id_数字"，不能重复。
+            edges还必须有一个label参数，其中label是输入中的直接词语或短语,edges中的source和target取自nodes中的id。
+            仅使用JSON进行响应，其格式可以在python中进行jsonify，并直接输入cy.add（data），包括“color”属性，以在前端显示图形。
+            您可以参考给定的示例：{json_example}。存储node和edge的数组中，最后一个元素后边不要有逗号，
+            确保边的目标和源与现有节点匹配。
+            不要在JSON的上方和下方包含markdown三引号，直接用花括号括起来。
+            """
+
+
+def generate_graph_info(raw_text: str) -> str | None:
+    """
+    generate graph info from raw text
+    :param raw_text:
+    :return:
+    """
+    messages = [
+        {"role": "system", "content": "你现在扮演信息抽取的角色，要求根据用户输入和AI的回答，正确提取出信息,记得不多对实体进行翻译。"},
+        {"role": "user", "content": raw_text},
+        {"role": "user", "content": __retriever_prompt}
+    ]
+    print("解析中....")
+    for i in range(3):
+        graph_info_result = llm.invoke(messages)
+        if len(graph_info_result)<10:
+            print("-------",i,"-------------------")
+            continue
+        else:
+            break
+    print(graph_info_result)
+    return graph_info_result
+
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+
+@app.route('/update_graph', methods=['POST'])
+def update_graph():
+    raw_text = request.json.get('text', '')
+    try:
+        result = generate_graph_info(raw_text)
+        if '```' in result:
+            graph_data=json.loads(result.split('```',2)[1].replace("json", ''))
+        else:
+            graph_data=json.loads(result)
+        return graph_data
+    except Exception as e:
+        return {'error': f"Error parsing graph data: {str(e)}"}
+
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=7860)

img/readme.txt

@@ -0,0 +1 @@
+1

rag/rag_class.py

@@ -0,0 +1,169 @@
+from langchain_community.vectorstores import Chroma,FAISS
+from langchain_community.llms import Ollama
+from langchain_core.output_parsers import StrOutputParser
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_core.runnables import RunnablePassthrough
+from operator import itemgetter
+from langchain.prompts import ChatPromptTemplate
+from rerank_code import rerank_topn
+from Config.config import VECTOR_DB,DB_directory
+from langchain_elasticsearch.vectorstores import ElasticsearchStore
+
+
+class RAG_class:
+    def __init__(self, model="qwen2:7b", embed="milkey/dmeta-embedding-zh:f16", c_name="sss1",
+                 persist_directory="E:/pycode/jupyter_code/langGraph/sss2/chroma.sqlite3/",es_url="http://localhost:9200"):
+        template = """
+        根据上下文回答以下问题,不要自己发挥，要根据以下参考内容总结答案，如果以下内容无法得到答案，就返回无法根据参考内容获取答案，
+
+        参考内容为：{context}
+
+        问题: {question}
+        """
+
+        self.prompts = ChatPromptTemplate.from_template(template)
+
+        # 使用 问题扩展+结果递归方式得到最终答案
+        template1 = """你是一个乐于助人的助手，可以生成与输入问题相关的多个子问题。
+        目标是将输入分解为一组可以单独回答的子问题/子问题。
+        生成多个与以下内容相关的搜索查询：{question}
+        输出4个相关问题,以换行符隔开："""
+        self.prompt_questions = ChatPromptTemplate.from_template(template1)
+
+        # 构建 问答对
+        template2 = """
+        以下是您需要回答的问题：
+
+        \n--\n {question} \n---\n
+
+        以下是任何可用的背景问答对：
+
+        \n--\n {q_a_pairs} \n---\n
+
+        以下是与该问题相关的其他上下文：
+
+        \n--\n {context} \n---\n
+
+        使用以上上下文和背景问答对来回答问题，问题是：{question} ，答案是：
+        """
+        self.decomposition_prompt = ChatPromptTemplate.from_template(template2)
+
+        self.llm = Ollama(model=model)
+        self.embeding = OllamaEmbeddings(model=embed)
+        if VECTOR_DB==1:
+            self.vectstore = Chroma(embedding_function=self.embeding, collection_name=c_name,
+                                persist_directory=persist_directory)
+        elif VECTOR_DB ==2:
+            self.vectstore = FAISS.load_local(folder_path=persist_directory + c_name, embeddings=self.embeding,
+                                            allow_dangerous_deserialization=True)
+        elif VECTOR_DB ==3:
+            self.vectstore = ElasticsearchStore(
+                es_url=es_url,
+                index_name=c_name,
+                embedding=self.embeding
+            )
+        self.retriever = self.vectstore.as_retriever()
+        try:
+            if VECTOR_DB==1:
+                self.vectstore = Chroma(embedding_function=self.embeding, collection_name=c_name,
+                                    persist_directory=persist_directory)
+            elif VECTOR_DB ==2:
+                self.vectstore = FAISS.load_local(folder_path=persist_directory + c_name, embeddings=self.embeding,
+                                               allow_dangerous_deserialization=True)
+            elif VECTOR_DB ==3:
+                self.vectstore = ElasticsearchStore(
+                    es_url=es_url,
+                    index_name=c_name,
+                    embedding=self.embeding
+                )
+            self.retriever = self.vectstore.as_retriever()
+        except Exception as e:
+            print("仅模型时无需加载数据库",e)
+    #
+    # Post-processing
+    def format_docs(self,docs):
+        return "\n\n".join(doc.page_content for doc in docs)
+    # 传统方式召回，单问题召回，然后llm总结答案回答
+    def simple_chain(self,question):
+        _chain = (
+            {"context": self.retriever|self.format_docs,"question":RunnablePassthrough()}
+            |self.prompts
+            |self.llm
+            |StrOutputParser()
+        )
+        answer = _chain.invoke({"question":question})
+        return answer
+
+    def rerank_chain(self,question):
+        retriever = self.vectstore.as_retriever(search_kwargs={"k": 10})
+        docs = retriever.invoke(question)
+        docs = rerank_topn(question,docs,N=5)
+        _chain = (
+                self.prompts
+                | self.llm
+                | StrOutputParser()
+        )
+        answer = _chain.invoke({"context":self.format_docs(docs),"question": question})
+        return answer
+
+    def format_qa_pairs(self, question, answer):
+        formatted_string = ""
+        formatted_string += f"Question: {question}\nAnswer:{answer}\n\n"
+        return formatted_string
+
+    # 获取问题的 扩展问题
+    def decomposition_chain(self, question):
+        _chain = (
+                {"question": RunnablePassthrough()}
+                | self.prompt_questions
+                | self.llm
+                | StrOutputParser()
+                | (lambda x: x.split("\n"))
+        )
+
+        questions = _chain.invoke({"question": question}) + [question]
+
+        return questions
+    # 多问题递归召回，每次召回后，问题和答案同时作为下一次召回的参考，再次用新问题召回
+    def rag_chain(self, questions):
+        q_a_pairs = ""
+        for q in questions:
+            _chain = (
+                    {"context": itemgetter("question") | self.retriever,
+                     "question": itemgetter("question"),
+                     "q_a_pairs": itemgetter("q_a_paris")
+                     }
+                    | self.decomposition_prompt
+                    | self.llm
+                    | StrOutputParser()
+            )
+
+            answer = _chain.invoke({"question": q, "q_a_paris": q_a_pairs})
+            q_a_pairs = self.format_qa_pairs(q, answer)
+            q_a_pairs = q_a_pairs + "\n----\n" + q_a_pairs
+        return answer
+
+    # 将聊天历史格式化为一个字符串
+    def format_chat_history(self,history):
+        formatted_history = ""
+        for role,content in history:
+            formatted_history += f"{role}: {content}\n"
+        return formatted_history
+    # 基于ollama大模型的大模型 多轮对话，不使用知识库的
+    def mult_chat(self,chat_history):
+        # 格式化聊天历史
+        formatted_history = self.format_chat_history(chat_history)
+
+        # 调用模型生成回复
+        response = self.llm.invoke(formatted_history)
+        return response
+
+
+
+# if __name__ == "__main__":
+#     rag = RAG_class(model="deepseek-r1:14b")
+#     question = "人卫社官网网址是？"
+#     questions = rag.decomposition_chain(question)
+#     print(questions)
+#     answer = rag.rag_chain(questions)
+#     print(answer)

rag/rerank_code.py

@@ -0,0 +1,21 @@
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained('E:\\model\\bge-reranker-large')
+model = AutoModelForSequenceClassification.from_pretrained('E:\\model\\bge-reranker-large')
+model.eval()
+
+
+def rerank_topn(question,docs,N=5):
+    pairs = []
+    for i in docs:
+        pairs.append([question,i.page_content])
+
+    with torch.no_grad():
+        inputs = tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=512)
+        scores = model(**inputs, return_dict=True).logits.view(-1, ).float()
+    scores = scores.argsort().numpy()[::-1][:N]
+    bk = []
+    for i in scores:
+        bk.append(docs[i])
+    return bk

requirements.txt

@@ -0,0 +1,10 @@
+gradio==4.29.0
+langchain-community==0.2.6
+langchain==0.2.6
+langchain-core==0.2.11
+requests
+transformers==4.41.1
+unstructured==0.7.12
+funasr==1.0.24
+modelscope
+chromadb

test/graph2neo4j.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+import sys
+sys.path.append(r"..//")#
+from Neo4j.neo4j_op import KnowledgeGraph
+from Neo4j.graph_extract import update_graph
+from Config.config import neo4j_host,neo4j_name,neo4j_pwd
+
+
+
+kg = KnowledgeGraph(neo4j_host,neo4j_name,neo4j_pwd)
+
+
+if __name__ == "__main__":
+
+    text = """范冰冰，1981年9月16日生于山东青岛，毕业于上海师范大学谢晋影视艺术学院，中国女演员，歌手。 
+    1998年参演电视剧《还珠格格》成名。2004年主演电影《手机》获得第27届大众电影百花奖最佳女演员奖。"""
+    res = update_graph(text)
+    # 批量创建节点
+    nodes = kg.create_nodes("node", res["nodes"])
+    print(nodes)
+    # 批量创建关系
+    relationships = kg.create_relationships([
+        ("node", {"name": edge["source"]}, "node", {"name": edge["target"]}, edge["label"]) for edge in res["edges"]
+    ])
+    print(relationships)