import gradio as gr from huggingface_hub import hf_hub_download from llama_cpp import Llama import sys MODEL_ID = "ibm-granite/granite-4.1-8b-GGUF" FILE_NAME = "granite-4.1-8b-Q4_K_M.gguf" print(f"Downloading {FILE_NAME} (~5GB)...", flush=True) model_path = hf_hub_download(repo_id=MODEL_ID, filename=FILE_NAME) print("Download complete!", flush=True) print("Loading model (this may take a minute)...", flush=True) llm = Llama( model_path=model_path, n_ctx=4096, n_threads=2, n_batch=512, verbose=False, ) print("Model loaded!", flush=True) def generate(prompt, max_tokens=512, temperature=0.7, top_p=0.9): output = llm.create_chat_completion( messages=[{"role": "user", "content": prompt}], max_tokens=max_tokens, temperature=temperature, top_p=top_p, ) return output["choices"][0]["message"]["content"] demo = gr.Interface( fn=generate, inputs=[ gr.Textbox(label="Prompt", lines=4), gr.Slider(64, 4096, value=512, step=64, label="Max Tokens"), gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature"), gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="Top P"), ], outputs=gr.Textbox(label="Resposta", lines=8), title="Granite 4.1 8B (CPU, Q4_K_M GGUF)", description="IBM Granite 4.1 8B em CPU com quantização Q4_K_M via GGUF. ~5GB RAM.", ) demo.launch()