stern-4.1-8b / app.py
tudob2b's picture
Use ibm-granite GGUF repo
8be27a5 verified
Raw History Blame Contribute Delete
1.39 kB
import gradio as gr
from huggingface_hub import hf_hub_download
from llama_cpp import Llama
import sys
MODEL_ID = "ibm-granite/granite-4.1-8b-GGUF"
FILE_NAME = "granite-4.1-8b-Q4_K_M.gguf"
print(f"Downloading {FILE_NAME} (~5GB)...", flush=True)
model_path = hf_hub_download(repo_id=MODEL_ID, filename=FILE_NAME)
print("Download complete!", flush=True)
print("Loading model (this may take a minute)...", flush=True)
llm = Llama(
model_path=model_path,
n_ctx=4096,
n_threads=2,
n_batch=512,
verbose=False,
)
print("Model loaded!", flush=True)
def generate(prompt, max_tokens=512, temperature=0.7, top_p=0.9):
output = llm.create_chat_completion(
messages=[{"role": "user", "content": prompt}],
max_tokens=max_tokens,
temperature=temperature,
top_p=top_p,
)
return output["choices"][0]["message"]["content"]
demo = gr.Interface(
fn=generate,
inputs=[
gr.Textbox(label="Prompt", lines=4),
gr.Slider(64, 4096, value=512, step=64, label="Max Tokens"),
gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature"),
gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="Top P"),
],
outputs=gr.Textbox(label="Resposta", lines=8),
title="Granite 4.1 8B (CPU, Q4_K_M GGUF)",
description="IBM Granite 4.1 8B em CPU com quantização Q4_K_M via GGUF. ~5GB RAM.",
)
demo.launch()