import os
from dotenv import load_dotenv
from typing import List, Dict, Any, Optional
import tempfile
import re
import json
import requests
from urllib.parse import urlparse
import pytesseract
from PIL import Image, ImageDraw, ImageFont, ImageEnhance, ImageFilter
import cmath
import pandas as pd
import uuid
import numpy as np
from code_interpreter import CodeInterpreter
import logging
interpreter_instance = CodeInterpreter()
from image_processing import *
from langchain_core.tools import tool
# import speech_recognition as sr
"""Langraph"""
from langgraph.graph import START, StateGraph, MessagesState
from langchain_community.tools.tavily_search import TavilySearchResults
from langchain_community.document_loaders import WikipediaLoader
from langchain_community.document_loaders import ArxivLoader
from langgraph.prebuilt import ToolNode, tools_condition
from langchain_google_genai import ChatGoogleGenerativeAI
from langchain_groq import ChatGroq
from langchain_huggingface import (
ChatHuggingFace,
HuggingFaceEndpoint,
HuggingFaceEmbeddings,
)
from langchain_community.vectorstores import SupabaseVectorStore
from langchain_core.messages import SystemMessage, HumanMessage
from langchain_core.tools import tool
from langchain.tools.retriever import create_retriever_tool
from supabase.client import Client, create_client
load_dotenv()
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("agent")
def tool_response(success: bool, data=None, error=None):
"""Standardized response format for tools."""
return {
"status": "success" if success else "error",
"data": data,
"error": error
}
from typing import Any
@tool
def multiply(a: float, b: float) -> float:
"""
Multiplies two numbers.
Args:
a (float): the first number
b (float): the second number
"""
logger.info("multiply called with a=%s, b=%s", a, b)
try:
a = float(a)
b = float(b)
result = a * b
return tool_response(True, result)
except Exception as e:
logger.error("multiply failed: %s", str(e))
return tool_response(False, error=f"Invalid input: {e}")
@tool
def add(a: float, b: float) -> float:
"""
Adds two numbers.
Args:
a (float): the first number
b (float): the second number
"""
logger.info("add called with a=%s, b=%s", a, b)
try:
a = float(a)
b = float(b)
return tool_response(True, a + b)
except Exception as e:
logger.error("add failed: %s", str(e))
return tool_response(False, error=f"Invalid input: {e}")
@tool
def subtract(a: float, b: float) -> int:
"""
Subtracts two numbers.
Args:
a (float): the first number
b (float): the second number
"""
logger.info("subtract called with a=%s, b=%s", a, b)
try:
a = float(a)
b = float(b)
return tool_response(True, a - b)
except Exception as e:
logger.error("subtract failed: %s", str(e))
return tool_response(False, error=f"Invalid input: {e}")
@tool
def divide(a: float, b: float) -> float:
"""
Divides two numbers.
Args:
a (float): the first float number
b (float): the second float number
"""
logger.info("divide called with a=%s, b=%s", a, b)
try:
a = float(a)
b = float(b)
if b == 0:
return tool_response(False, error="Division by zero")
return tool_response(True, a / b)
except Exception as e:
logger.error("divide failed: %s", str(e))
return tool_response(False, error=f"Invalid input: {e}")
@tool
def modulus(a: int, b: int) -> int:
"""
Get the modulus of two numbers.
Args:
a (int): the first number
b (int): the second number
"""
logger.info("modulus called with a=%s, b=%s", a, b)
try:
a = int(a)
b = int(b)
return tool_response(True, a % b)
except Exception as e:
logger.error("modulus failed: %s", str(e))
return tool_response(False, error=f"Invalid input: {e}")
@tool
def power(a: float, b: float) -> float:
"""
Get the power of two numbers.
Args:
a (float): the first number
b (float): the second number
"""
logger.info("power called with a=%s, b=%s", a, b)
try:
a = float(a)
b = float(b)
return tool_response(True, a ** b)
except Exception as e:
logger.error("power failed: %s", str(e))
return tool_response(False, error=f"Invalid input: {e}")
@tool
def square_root(a: float) -> float|complex:
"""
Get the square root of a number.
Args:
a (float): the number to get the square root of
"""
logger.info("square_root called with a=%s", a)
try:
a = float(a)
if a < 0:
# use complex math if negative
return tool_response(True, str(cmath.sqrt(a)))
return tool_response(True, a ** 0.5)
except Exception as e:
logger.error("square_root failed: %s", str(e))
return tool_response(False, error=f"Invalid input: {e}")
# =========================
# 📂 File Tools
# =========================
@tool
def save_and_read_file(filename: str, content: str):
"""
Save content to a file and return the path.
Args:
content (str): the content to save to the file
filename (str, optional): the name of the file. If not provided, a random name file will be created.
"""
logger.info("save_and_read_file called with filename=%s", filename)
try:
with open(filename, "w", encoding="utf-8") as f:
f.write(content)
with open(filename, "r", encoding="utf-8") as f:
result = f.read()
return tool_response(True, result)
except Exception as e:
logger.error("save_and_read_file failed: %s", str(e))
return tool_response(False, error=f"File error: {e}")
@tool
def download_file_from_url(url: str):
"""
Download a file from a URL and save it to a temporary location.
Args:
url (str): the URL of the file to download.
filename (str, optional): the name of the file. If not provided, a random name file will be created.
"""
logger.info("download_file_from_url called with url=%s", url)
try:
if url.startswith("file://"):
raise ValueError("Local file:// URLs not allowed")
response = requests.get(url, timeout=10)
response.raise_for_status()
filename = os.path.basename(urlparse(url).path) or f"download_{uuid.uuid4()}"
with open(filename, "wb") as f:
f.write(response.content)
return tool_response(True, filename)
except Exception as e:
logger.error("download_file_from_url failed: %s", str(e))
return tool_response(False, error=f"Download error: {e}")
# =========================
# 🖼️ Image Tools
# =========================
@tool
def extract_text_from_image(image_path: str) -> str:
"""
Extract text from an image using OCR library pytesseract (if available).
Args:
image_path (str): the path to the image file.
"""
try:
# Open the image
image = Image.open(image_path)
# Extract text from the image
text = pytesseract.image_to_string(image)
return f"Extracted text from image:\n\n{text}"
except Exception as e:
return f"Error extracting text from image: {str(e)}"
@tool
def analyze_image(image_base64: str) -> Dict[str, Any]:
"""
Analyze basic properties of an image (size, mode, color analysis, thumbnail preview).
Args:
image_base64 (str): Base64 encoded image string
Returns:
Dictionary with analysis result
"""
try:
img = decode_image(image_base64)
width, height = img.size
mode = img.mode
if mode in ("RGB", "RGBA"):
arr = np.array(img)
avg_colors = arr.mean(axis=(0, 1))
dominant = ["Red", "Green", "Blue"][np.argmax(avg_colors[:3])]
brightness = avg_colors.mean()
color_analysis = {
"average_rgb": avg_colors.tolist(),
"brightness": brightness,
"dominant_color": dominant,
}
else:
color_analysis = {"note": f"No color analysis for mode {mode}"}
thumbnail = img.copy()
thumbnail.thumbnail((100, 100))
thumb_path = save_image(thumbnail, "thumbnails")
thumbnail_base64 = encode_image(thumb_path)
return {
"dimensions": (width, height),
"mode": mode,
"color_analysis": color_analysis,
"thumbnail": thumbnail_base64,
}
except Exception as e:
return {"error": str(e)}
@tool
def transform_image(
image_base64: str, operation: str, params: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
"""
Apply transformations: resize, rotate, crop, flip, brightness, contrast, blur, sharpen, grayscale.
Args:
image_base64 (str): Base64 encoded input image
operation (str): Transformation operation
params (Dict[str, Any], optional): Parameters for the operation
Returns:
Dictionary with transformed image (base64)
"""
try:
img = decode_image(image_base64)
params = params or {}
if operation == "resize":
img = img.resize(
(
params.get("width", img.width // 2),
params.get("height", img.height // 2),
)
)
elif operation == "rotate":
img = img.rotate(params.get("angle", 90), expand=True)
elif operation == "crop":
img = img.crop(
(
params.get("left", 0),
params.get("top", 0),
params.get("right", img.width),
params.get("bottom", img.height),
)
)
elif operation == "flip":
if params.get("direction", "horizontal") == "horizontal":
img = img.transpose(Image.FLIP_LEFT_RIGHT)
else:
img = img.transpose(Image.FLIP_TOP_BOTTOM)
elif operation == "adjust_brightness":
img = ImageEnhance.Brightness(img).enhance(params.get("factor", 1.5))
elif operation == "adjust_contrast":
img = ImageEnhance.Contrast(img).enhance(params.get("factor", 1.5))
elif operation == "blur":
img = img.filter(ImageFilter.GaussianBlur(params.get("radius", 2)))
elif operation == "sharpen":
img = img.filter(ImageFilter.SHARPEN)
elif operation == "grayscale":
img = img.convert("L")
else:
return {"error": f"Unknown operation: {operation}"}
result_path = save_image(img)
result_base64 = encode_image(result_path)
return {"transformed_image": result_base64}
except Exception as e:
return {"error": str(e)}
@tool
def draw_on_image(
image_base64: str, drawing_type: str, params: Dict[str, Any]
) -> Dict[str, Any]:
"""
Draw shapes (rectangle, circle, line) or text onto an image.
Args:
image_base64 (str): Base64 encoded input image
drawing_type (str): Drawing type
params (Dict[str, Any]): Drawing parameters
Returns:
Dictionary with result image (base64)
"""
try:
img = decode_image(image_base64)
draw = ImageDraw.Draw(img)
color = params.get("color", "red")
if drawing_type == "rectangle":
draw.rectangle(
[params["left"], params["top"], params["right"], params["bottom"]],
outline=color,
width=params.get("width", 2),
)
elif drawing_type == "circle":
x, y, r = params["x"], params["y"], params["radius"]
draw.ellipse(
(x - r, y - r, x + r, y + r),
outline=color,
width=params.get("width", 2),
)
elif drawing_type == "line":
draw.line(
(
params["start_x"],
params["start_y"],
params["end_x"],
params["end_y"],
),
fill=color,
width=params.get("width", 2),
)
elif drawing_type == "text":
font_size = params.get("font_size", 20)
try:
font = ImageFont.truetype("arial.ttf", font_size)
except IOError:
font = ImageFont.load_default()
draw.text(
(params["x"], params["y"]),
params.get("text", "Text"),
fill=color,
font=font,
)
else:
return {"error": f"Unknown drawing type: {drawing_type}"}
result_path = save_image(img)
result_base64 = encode_image(result_path)
return {"result_image": result_base64}
except Exception as e:
return {"error": str(e)}
@tool
def generate_simple_image(
image_type: str,
width: int = 500,
height: int = 500,
params: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
Generate a simple image (gradient, noise, pattern, chart).
Args:
image_type (str): Type of image
width (int), height (int)
params (Dict[str, Any], optional): Specific parameters
Returns:
Dictionary with generated image (base64)
"""
try:
params = params or {}
if image_type == "gradient":
direction = params.get("direction", "horizontal")
start_color = params.get("start_color", (255, 0, 0))
end_color = params.get("end_color", (0, 0, 255))
img = Image.new("RGB", (width, height))
draw = ImageDraw.Draw(img)
if direction == "horizontal":
for x in range(width):
r = int(
start_color[0] + (end_color[0] - start_color[0]) * x / width
)
g = int(
start_color[1] + (end_color[1] - start_color[1]) * x / width
)
b = int(
start_color[2] + (end_color[2] - start_color[2]) * x / width
)
draw.line([(x, 0), (x, height)], fill=(r, g, b))
else:
for y in range(height):
r = int(
start_color[0] + (end_color[0] - start_color[0]) * y / height
)
g = int(
start_color[1] + (end_color[1] - start_color[1]) * y / height
)
b = int(
start_color[2] + (end_color[2] - start_color[2]) * y / height
)
draw.line([(0, y), (width, y)], fill=(r, g, b))
elif image_type == "noise":
noise_array = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8)
img = Image.fromarray(noise_array, "RGB")
else:
return {"error": f"Unsupported image_type {image_type}"}
result_path = save_image(img)
result_base64 = encode_image(result_path)
return {"generated_image": result_base64}
except Exception as e:
return {"error": str(e)}
@tool
def combine_images(
images_base64: List[str], operation: str, params: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
"""
Combine multiple images (collage, stack, blend).
Args:
images_base64 (List[str]): List of base64 images
operation (str): Combination type
params (Dict[str, Any], optional)
Returns:
Dictionary with combined image (base64)
"""
try:
images = [decode_image(b64) for b64 in images_base64]
params = params or {}
if operation == "stack":
direction = params.get("direction", "horizontal")
if direction == "horizontal":
total_width = sum(img.width for img in images)
max_height = max(img.height for img in images)
new_img = Image.new("RGB", (total_width, max_height))
x = 0
for img in images:
new_img.paste(img, (x, 0))
x += img.width
else:
max_width = max(img.width for img in images)
total_height = sum(img.height for img in images)
new_img = Image.new("RGB", (max_width, total_height))
y = 0
for img in images:
new_img.paste(img, (0, y))
y += img.height
else:
return {"error": f"Unsupported combination operation {operation}"}
result_path = save_image(new_img)
result_base64 = encode_image(result_path)
return {"combined_image": result_base64}
except Exception as e:
return {"error": str(e)}
# =========================
# 📊 Data Tools
# =========================
@tool
def analyze_csv_file(file_path: str):
"""
Analyze a CSV file using pandas and answer a question about it.
Args:
file_path (str): the path to the CSV file.
query (str): Question about the data
"""
logger.info("analyze_csv_file called with file_path=%s", file_path)
try:
df = pd.read_csv(file_path)
summary = {"shape": df.shape, "columns": df.columns.tolist(), "head": df.head(3).to_dict()}
return tool_response(True, summary)
except Exception as e:
logger.error("analyze_csv_file failed: %s", str(e))
return tool_response(False, error=f"CSV analysis error: {e}")
@tool
def analyze_excel_file(file_path: str):
"""
Analyze an Excel file using pandas and answer a question about it.
Args:
file_path (str): the path to the Excel file.
query (str): Question about the data
"""
logger.info("analyze_excel_file called with file_path=%s", file_path)
try:
df = pd.read_excel(file_path)
summary = {"shape": df.shape, "columns": df.columns.tolist(), "head": df.head(3).to_dict()}
return tool_response(True, summary)
except Exception as e:
logger.error("analyze_excel_file failed: %s", str(e))
return tool_response(False, error=f"Excel analysis error: {e}")
# =========================
# 💻 Code Tool
# =========================
@tool
def execute_code_multilang(code: str, language: str = "python") -> str:
"""Execute code in multiple languages (Python, Bash, SQL, C, Java) and return results.
Args:
code (str): The source code to execute.
language (str): The language of the code. Supported: "python", "bash", "sql", "c", "java".
Returns:
A string summarizing the execution results (stdout, stderr, errors, plots, dataframes if any).
"""
supported_languages = ["python", "bash", "sql", "c", "java"]
language = language.lower()
if language not in supported_languages:
return f"❌ Unsupported language: {language}. Supported languages are: {', '.join(supported_languages)}"
result = interpreter_instance.execute_code(code, language=language)
response = []
if result["status"] == "success":
response.append(f"✅ Code executed successfully in **{language.upper()}**")
if result.get("stdout"):
response.append(
"\n**Standard Output:**\n```\n" + result["stdout"].strip() + "\n```"
)
if result.get("stderr"):
response.append(
"\n**Standard Error (if any):**\n```\n"
+ result["stderr"].strip()
+ "\n```"
)
if result.get("result") is not None:
response.append(
"\n**Execution Result:**\n```\n"
+ str(result["result"]).strip()
+ "\n```"
)
if result.get("dataframes"):
for df_info in result["dataframes"]:
response.append(
f"\n**DataFrame `{df_info['name']}` (Shape: {df_info['shape']})**"
)
df_preview = pd.DataFrame(df_info["head"])
response.append("First 5 rows:\n```\n" + str(df_preview) + "\n```")
if result.get("plots"):
response.append(
f"\n**Generated {len(result['plots'])} plot(s)** (Image data returned separately)"
)
else:
response.append(f"❌ Code execution failed in **{language.upper()}**")
if result.get("stderr"):
response.append(
"\n**Error Log:**\n```\n" + result["stderr"].strip() + "\n```"
)
return "\n".join(response)
# =========================
# 🌍 Search Tools
# =========================
@tool
def web_search(query: str) -> str:
"""Search Tavily for a query and return maximum 3 results.
Args:
query: The search query."""
search_docs = TavilySearchResults(max_results=3).invoke(query)
formatted_search_docs = "\n\n---\n\n".join(
[
f'\n{doc.get("content", "")}\n'
for doc in search_docs
]
)
return {"web_results": formatted_search_docs}
@tool
def wiki_search(query: str) -> str:
"""Search Wikipedia for a query and return maximum 2 results.
Args:
query: The search query."""
search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
formatted_search_docs = "\n\n---\n\n".join(
[
f'\n{doc.page_content}\n'
for doc in search_docs
]
)
return {"wiki_results": formatted_search_docs}
@tool
def arxiv_search(query: str) -> str:
"""Search Arxiv for a query and return maximum 3 result.
Args:
query: The search query."""
search_docs = ArxivLoader(query=query, load_max_docs=3).load()
formatted_search_docs = "\n\n---\n\n".join(
[
f'\n{doc.page_content[:1000]}\n'
for doc in search_docs
]
)
return {"arxiv_results": formatted_search_docs}
# =========================
# 🌍 Tested for tools
# =========================
# 🌍 Search Tools
# print("\n--- web_search ---")
# print(web_search.invoke({"query": "latest AI research", "max_results": 2}))
# print("\n--- wiki_search ---")
# print(wiki_search.invoke({"query": "LangChain"}))
# print("\n--- arxiv_search ---")
# print(arxiv_search.invoke({"query": "transformers"}))
# 💻 Code Execution
# print("\n--- execute_code_multilang ---")
# print(execute_code_multilang.invoke({"code": "print(2+3)", "language": "python"}))
# load the system prompt from the file
with open("system_prompt.txt", "r", encoding="utf-8") as f:
system_prompt = f.read()
print(system_prompt)
# System message
sys_msg = SystemMessage(content=system_prompt)
# build a retriever
embeddings = HuggingFaceEmbeddings(
model_name="sentence-transformers/all-mpnet-base-v2"
) # dim=768
from dotenv import load_dotenv
load_dotenv()
supabase_url = os.environ.get("SUPABASE_URL")
supabase_key = os.environ.get("SUPABASE_KEY")
supabase: Client = create_client(
supabase_url, supabase_key
)
vector_store = SupabaseVectorStore(
client=supabase,
embedding=embeddings,
table_name="documents2",
query_name="match_documents_2",
)
create_retriever_tool = create_retriever_tool(
retriever=vector_store.as_retriever(),
name="Question Search",
description="A tool to retrieve similar questions from a vector store.",
)
tools = [
web_search,
wiki_search,
arxiv_search,
multiply,
add,
subtract,
divide,
modulus,
power,
square_root,
save_and_read_file,
download_file_from_url,
extract_text_from_image,
analyze_csv_file,
analyze_excel_file,
execute_code_multilang,
analyze_image,
transform_image,
draw_on_image,
generate_simple_image,
combine_images,
]
# Build graph function
def build_graph(provider: str = "groq"):
"""Build the graph"""
# Load environment variables from .env file
import time
import httpx
class ChatGroqWithRetry(ChatGroq):
def invoke(self, *args, **kwargs):
max_retries = 5
for attempt in range(max_retries):
try:
return super().invoke(*args, **kwargs)
except httpx.HTTPStatusError as e:
if e.response.status_code == 429:
wait = min(2 ** attempt, 30)
time.sleep(wait)
continue
raise
raise Exception("Groq API: Too Many Requests after retries.")
if provider == "groq":
# Groq https://console.groq.com/docs/models
llm = ChatGroqWithRetry(model="qwen/qwen3-32b", temperature=0)
elif provider == "huggingface":
# TODO: Add huggingface endpoint
llm = ChatHuggingFace(
llm=HuggingFaceEndpoint(
repo_id="TinyLlama/TinyLlama-1.1B-Chat-v1.0",
task="text-generation", # for chat‐style use “text-generation”
max_new_tokens=1024,
do_sample=False,
repetition_penalty=1.03,
temperature=0,
),
verbose=True,
)
else:
raise ValueError("Invalid provider. Choose 'groq' or 'huggingface'.")
# Bind tools to LLM
llm_with_tools = llm.bind_tools(tools)
# Node
def assistant(state: MessagesState):
"""Assistant node"""
return {"messages": [llm_with_tools.invoke(state["messages"])]}
def retriever(state: MessagesState):
"""Retriever node"""
similar_question = vector_store.similarity_search(state["messages"][0].content)
if similar_question: # Check if the list is not empty
example_msg = HumanMessage(
content=f"Here I provide a similar question and answer for reference: \n\n{similar_question[0].page_content}",
)
return {"messages": [sys_msg] + state["messages"] + [example_msg]}
else:
# Handle the case when no similar questions are found
return {"messages": [sys_msg] + state["messages"]}
builder = StateGraph(MessagesState)
builder.add_node("retriever", retriever)
builder.add_node("assistant", assistant)
builder.add_node("tools", ToolNode(tools))
builder.add_edge(START, "retriever")
builder.add_edge("retriever", "assistant")
builder.add_conditional_edges(
"assistant",
tools_condition,
)
builder.add_edge("tools", "assistant")
# Compile graph
return builder.compile()
# test
# if __name__ == "__main__":
# question = "When was a picture of St. Thomas Aquinas first added to the Wikipedia page on the Principle of double effect?"
# graph = build_graph(provider="groq")
# messages = [HumanMessage(content=question)]
# messages = graph.invoke({"messages": messages})
# for m in messages["messages"]:
# m.pretty_print()