| import spaces | |
| import os | |
| import json | |
| import subprocess | |
| from functools import lru_cache | |
| from cachetools import LRUCache | |
| import threading | |
| MODEL_CACHE = LRUCache(maxsize=3) # max 10 models | |
| CACHE_LOCK = threading.Lock() | |
| from docling.document_converter import DocumentConverter | |
| ### monkey patch | |
| # from llama_cpp.llama_chat_format import Qwen35ChatHandler | |
| # import llama_cpp._internals as internals | |
| # 2️⃣ Monkey patch BEFORE creating Llama() | |
| # _original_close = internals.LlamaModel.close | |
| def safe_close(self): | |
| try: | |
| if hasattr(self, "sampler") and self.sampler is not None: | |
| return _original_close(self) | |
| except Exception: | |
| pass | |
| # internals.LlamaModel.close = safe_close | |
| def safe_del(self): | |
| try: | |
| self.close() | |
| except Exception: | |
| pass | |
| # internals.LlamaModel.__del__ = safe_del | |
| ##### final verdict | |
| # qwen 3 next 80b infrence | |
| #qwen 3 VL | |
| #mini max 2.5 | |
| #gpt oss 120B #seems unstable | |
| # import llama_cpp._internals as internals | |
| # from llama_cpp.llama_chat_format import Qwen3VLChatHandler | |
| # from llama_cpp import Llama | |
| # from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType | |
| # from llama_cpp_agent.providers import LlamaCppPythonProvider | |
| # from llama_cpp_agent.chat_history import BasicChatHistory | |
| # from llama_cpp_agent.chat_history.messages import Roles | |
| # import gradio as gr | |
| # from huggingface_hub import hf_hub_download | |
| # import llama_cpp | |
| # print(llama_cpp.__file__) | |
| # print(llama_cpp.__version__) | |
| huggingface_token = os.getenv("HUGGINGFACE_TOKEN") | |
| # hf_hub_download( | |
| # repo_id="bartowski/gemma-2-9b-it-GGUF", | |
| # filename="gemma-2-9b-it-Q5_K_M.gguf", | |
| # local_dir="./models" | |
| # ) | |
| # hf_hub_download( | |
| # repo_id="bartowski/gemma-2-27b-it-GGUF", | |
| # filename="gemma-2-27b-it-Q5_K_M.gguf", | |
| # local_dir="./models" | |
| # ) | |
| # hf_hub_download( | |
| # repo_id="google/gemma-2-2b-it-GGUF", | |
| # filename="2b_it_v2.gguf", | |
| # local_dir="./models", | |
| # token=huggingface_token | |
| # ) | |
| # hf_hub_donload( | |
| # repo_id="unsloth/GLM-4.7-Flash-GGUF", | |
| # filename="GLM-4.7-Flash-UD-Q8_K_XL.gguf", | |
| # local_dir="./models", | |
| # token=huggingface_token | |
| # ) | |
| # hf_hub_download( | |
| # repo_id="unsloth/gpt-oss-20b-GGUF", | |
| # filename="gpt-oss-20b-Q4_K_M.gguf", | |
| # local_dir="./models", | |
| # token=huggingface_token | |
| # ) | |
| # hf_hub_download( | |
| # repo_id="unsloth/gpt-oss-20b-GGUF", | |
| # filename="gpt-oss-20b-Q4_K_M.gguf", | |
| # local_dir="./models", | |
| # token=huggingface_token | |
| # ) | |
| # hf_hub_download( | |
| # repo_id="unsloth/Qwen3-Next-80B-A3B-Instruct-GGUF", | |
| # filename="Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf", | |
| # local_dir="./models", | |
| # token=huggingface_token | |
| # ) | |
| # hf_hub_download( | |
| # repo_id="unsloth/Qwen3-VL-32B-Thinking-GGUF", | |
| # filename="Qwen3-VL-32B-Thinking-Q8_0.gguf", | |
| # local_dir="./models" | |
| # ) | |
| # hf_hub_download( | |
| # repo_id="unsloth/Qwen3-Coder-Next-GGUF", | |
| # filename="Qwen3-Coder-Next-Q4_K_M.gguf", | |
| # local_dir="./models" | |
| # ) | |
| import os | |
| import base64 | |
| import os | |
| import tempfile | |
| import requests | |
| from urllib.parse import urlparse | |
| import os | |
| llm = None | |
| llm_model = None | |
| import json | |
| def str_to_json(str_obj): | |
| json_obj = json.loads(str_obj) | |
| return json_obj | |
| import os | |
| def a(): | |
| print() | |
| # | |
| def respond( | |
| message, | |
| history: list[tuple[str, str]], | |
| model, | |
| system_message, | |
| max_tokens, | |
| temperature, | |
| top_p, | |
| top_k, | |
| repeat_penalty, | |
| ): | |
| try: | |
| j_r= str_to_json(message) | |
| try: | |
| mode = j_r['mode'] | |
| except: | |
| yield "bekar request " | |
| except Exception as e: | |
| print(e) | |
| print("not valid json") | |
| if mode=="1": | |
| source= j_r["source"] | |
| # source = "https://arxiv.org/pdf/2408.09869" # a document via a local path or URL | |
| converter = DocumentConverter() | |
| result = converter.convert(source) | |
| print(result.document.export_to_markdown()) | |
| yield str(x) | |
| description = """<p align="center">Defaults to 2B (you can switch to 9B or 27B from additional inputs)</p> | |
| <p><center> | |
| <a href="https://huggingface.co/google/gemma-2-27b-it" target="_blank">[27B it Model]</a> | |
| <a href="https://huggingface.co/google/gemma-2-9b-it" target="_blank">[9B it Model]</a> | |
| <a href="https://huggingface.co/google/gemma-2-2b-it" target="_blank">[2B it Model]</a> | |
| <a href="https://huggingface.co/bartowski/gemma-2-27b-it-GGUF" target="_blank">[27B it Model GGUF]</a> | |
| <a href="https://huggingface.co/bartowski/gemma-2-9b-it-GGUF" target="_blank">[9B it Model GGUF]</a> | |
| <a href="https://huggingface.co/google/gemma-2-2b-it-GGUF" target="_blank">[2B it Model GGUF]</a> | |
| </center></p> | |
| """ | |
| import gradio as gr | |
| demo = gr.ChatInterface( | |
| fn=respond, | |
| additional_inputs=[ | |
| gr.Dropdown( | |
| [ | |
| # "gemma-2-9b-it-Q5_K_M.gguf", | |
| # "gemma-2-27b-it-Q5_K_M.gguf", | |
| # # "2b_it_v2.gguf", | |
| # "GLM-4.7-Flash-UD-Q8_K_XL.gguf", | |
| # "Qwen3-Coder-Next-Q4_K_M.gguf", | |
| # "gpt-oss-20b-Q4_K_M.gguf", | |
| # "Qwen3-Next-80B-A3B-Instruct-Q4_K_M.gguf", | |
| "UD-Q3_K_XL/Qwen3.5-122B-A10B-UD-Q3_K_XL-00001-of-00003.gguf", | |
| # "Qwen3-VL-32B-Thinking-Q8_0.gguf", | |
| # "Q8_0/gpt-oss-120b-Q8_0-00001-of-00002.gguf" | |
| ], | |
| value= "UD-Q3_K_XL/Qwen3.5-122B-A10B-UD-Q3_K_XL-00001-of-00003.gguf", | |
| label="Model", | |
| ), | |
| gr.Textbox( | |
| value="You are a helpful assistant.", | |
| label="System message", | |
| ), | |
| gr.Slider(1, 4096, value=2048, step=1, label="Max tokens"), | |
| gr.Slider(0.1, 4.0, value=0.7, step=0.1, label="Temperature"), | |
| gr.Slider(0.1, 1.0, value=0.95, step=0.05, label="Top-p"), | |
| gr.Slider(0, 100, value=40, step=1, label="Top-k"), | |
| gr.Slider(0.0, 2.0, value=1.1, step=0.1, label="Repetition penalty"), | |
| ], | |
| title="Chat with Gemma 2 using llama.cpp", | |
| description=description, | |
| ) | |
| # demo.launch() | |
| if __name__ == "__main__": | |
| demo.launch() |