Update app.py

app.py

@@ -83,14 +83,25 @@ math_tool = FunctionTool.from_defaults(
 TOOLS = [validate_tool, search_tool, math_tool]
 
 from llama_index.core.agent import ReActAgent
-from llama_index.llms.huggingface_api import HuggingFaceInferenceAPI
-
-llm = HuggingFaceInferenceAPI(
+# from llama_index.llms.huggingface_api import HuggingFaceInferenceAPI
+
+# llm = HuggingFaceInferenceAPI(
+#     model_name="mistralai/Mistral-7B-Instruct-v0.3",
+#     token=HF_TOKEN,
+#     context_window=3900,
+#     max_new_tokens=256,
+#     generate_kwargs={"temperature": 0.7}
+# )
+
+from llama_index.llms import HuggingFaceLLM
+
+llm = HuggingFaceLLM(
+    context_window=4096,
+    max_new_tokens=512,
+    generate_kwargs={"temperature": 0.7, "top_p": 0.95},
+    tokenizer_name="mistralai/Mistral-7B-Instruct-v0.3",
     model_name="mistralai/Mistral-7B-Instruct-v0.3",
-    token=HF_TOKEN,
-    context_window=3900,
-    max_new_tokens=256,
-    generate_kwargs={"temperature": 0.7}
+    device="cpu",  # or "cuda" if available
 )
 
 agent = ReActAgent.from_tools(