Omni Capabilities

pip install "transformers>=5.20.2" accelerate datasets torchcodec pillow librosa soundfile

import torch
from transformers import AutoModelForMultimodalLM, AutoProcessor

model_id = "LiquidAI/LFM3-Omni-2B"
device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" else torch.float32

model = AutoModelForMultimodalLM.from_pretrained(
    model_id, dtype=dtype, device_map=device, trust_remote_code=True
).eval()
processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
codec = processor.audio_tokenizer.to(device=device).eval()

import soundfile as sf

SR = 24_000  # generated speech is 24 kHz


@torch.inference_mode()
def generate(conversation, generation_mode=None, max_new_tokens=512, **processor_kwargs):
    inputs = processor.apply_chat_template(
        conversation,
        add_generation_prompt=True,
        tokenize=True,
        return_dict=True,
        return_tensors="pt",
        processor_kwargs=processor_kwargs,
    ).to(model.device)
    out = model.generate(
        **inputs,
        max_new_tokens=max_new_tokens,
        generation_mode=generation_mode,
        audio_temperature=0.4,
        audio_repetition_penalty=1.2,
        audio_repetition_window=64,
    )
    prompt_len = inputs["input_ids"].shape[1]
    reply = processor.decode(out.sequences[0, prompt_len : out.sequence_lengths[0]], skip_special_tokens=True)
    speech = None
    if out.audio_lengths[0] > 0:
        speech = codec.decode(out.audio_codes[:1, :, : out.audio_lengths[0]]).audio_values[0].float().cpu().numpy().squeeze()
    return reply, speech

torch.manual_seed(0)
_, speech = generate([
    {"role": "system", "content": "Perform TTS in American English using a female voice."},
    {"role": "user", "content": "What is this tiny garden visitor doing?"},
], generation_mode="audio")
sf.write("question.wav", speech, SR)

_, speech_de = generate([
    {"role": "system", "content": "Perform TTS in German using a male voice."},
    {"role": "user", "content": "Heute probieren wir etwas Neues: Wir malen Tiere mit geschlossenen Augen. Mein erster Versuch sieht aus wie eine Kartoffel mit Ohren."},
], generation_mode="audio")
sf.write("german.wav", speech_de, SR)

for path, language in [("question.wav", "English"), ("german.wav", "German")]:
    transcript, _ = generate([
        {"role": "system", "content": f"Perform ASR in {language}."},
        {"role": "user", "content": [{"type": "audio", "audio": path}]},
    ])
    print(f"[{language}] {transcript}")

[English] What is this tiny garden visitor doing?
[German] Heute probieren wir etwas Neues. Wir malen Tiere mit geschlossenen Augen. Mein erster Versuch sieht aus wie eine Kartoffel mit Ohren.

translation, _ = generate([
    {"role": "system", "content": "Convert the speech in German to written English."},
    {"role": "user", "content": [{"type": "audio", "audio": "german.wav"}]},
])
print("Translation:", translation)

Translation: Today we are trying something new. We paint animals with closed eyes. My first attempt looks like a potato with ears.

reply, speech = generate(
    [
        {"role": "system", "content": "Respond with interleaved text and audio."},
        {"role": "user", "content": "I want to move more, but I have not found an activity I enjoy. How can I try a few options and build a small habit? Give me two practical ideas in short, friendly sentences."},
    ],
    max_new_tokens=1024,
)
print(reply)
sf.write("reply.wav", speech, SR)

Try a short walk in a new neighborhood each day and notice how it feels. Then pick a weekly community class that interests you and attend for a few sessions.

reply, _ = generate([{"role": "user", "content": "Write a haiku about a small model that can see and hear."}])
print(reply)

Tiny lens, keen eye,
Hear the world in whispers,
Small model, awake.

from transformers.image_utils import load_image

image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg")
image.save("bee.jpg")

caption, _ = generate([
    {"role": "system", "content": "Describe the provided image."},
    {"role": "user", "content": [{"type": "image", "image": "bee.jpg"}]},
])
print(caption)

This is a close‑up (macro) photograph of a garden scene dominated by pink cosmos flowers. The central subject is a fully open, light‑pink cosmos with five broad, slightly ruffled petals radiating from a bright yellow disc of stamens; perched right in the middle of that disc is a fuzzy bumblebee, black with a band of orange‑yellow on its thorax and pale, hairy wings, actively foraging on the orange center. To the left, a second, similar pink cosmos is in soft focus, its yellow center just visible, and behind it are several dried, brown, spent flower heads with withered petals. In the lower right, a small, vivid red cosmos blooms with its five pointed petals, accompanied by a small pale, withered bud on a thin reddish stem. The background is rendered in smooth, out‑of‑focus green, with a large, broad green leaf arcing across the top of the frame and finer green foliage filling the lower area, giving the image a shallow depth of field that isolates the pollinating bee and the pink bloom in sharp detail.

answer, _ = generate([
    {"role": "system", "content": "Answer questions about the provided image. Base every answer only on what can be seen. Respond with text."},
    {"role": "user", "content": [{"type": "image", "image": "bee.jpg"}, {"type": "audio", "audio": "question.wav"}]},
])
print(answer)

The tiny garden visitor is a **bee** (a bumblebee, identifiable by its black body with yellow-orange bands and fuzzy white abdomen) perched on the center of the pink cosmos flower. It is **feeding on the flower's nectar** — its head is positioned directly over the orange, pollen‑bearing center of the bloom, with its antennae extended toward the stamens as it collects nectar. In short, the bee is visiting the flower to feed on its nectar.

answer, speech = generate([
    {"role": "system", "content": "Answer questions about the provided image. Base every answer only on what can be seen. Respond with interleaved text and audio."},
    {"role": "user", "content": [{"type": "image", "image": "bee.jpg"}, {"type": "text", "text": "Is this insect useful for the garden? Why?"}]},
])
print(answer)
sf.write("bee_answer.wav", speech, SR)

Yes, this insect is useful for the garden because it is a bee, and bees pollinate flowers, helping plants reproduce and grow.

from huggingface_hub import hf_hub_download

video = hf_hub_download("raushan-testing-hf/videos-test", "sample_demo_1.mp4", repo_type="dataset")


@torch.inference_mode()
def ask_video(question, use_audio=False):
    extra = dict(load_audio_from_video=True, processor_kwargs={"use_audio_in_video": True}) if use_audio else {}
    inputs = processor.apply_chat_template(
        [
            {"role": "system", "content": "Answer questions about the video using what you see and hear."},
            {"role": "user", "content": [{"type": "video", "video": video}, {"type": "text", "text": question}]},
        ],
        add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", **extra,
    ).to(model.device)
    out = model.generate(**inputs, max_new_tokens=256, generation_mode="text")
    return processor.decode(out.sequences[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True)


print("Frames only:\n", ask_video("What happens in this video?"))
print("\nFrames and soundtrack:\n", ask_video("What happens in this video, and what can you hear?", use_audio=True))

The video shows a young toddler sitting on a bed and playing with an open book. The child, who has blonde hair, is wearing dark-framed glasses, a light blue sleeveless top, and pink pants, and is barefoot with legs stretched out on the bed. At the start, the child is looking down at the open book, then raises both hands and claps them together in a playful gesture. After a moment, the child lowers their hands, grabs the book with both hands, and begins turning the pages, flipping through the pages while continuing to look at them. The setting is a bedroom, with a crib visible in the background, a white blanket and clothing scattered on the bed, and a small framed photo on a surface to the right. A "clideo.com" watermark appears in the bottom right corner throughout the clip.

answer, _ = generate([
    {"role": "user", "content": [
        {"type": "audio", "audio": "german.wav"},
        {"type": "text", "text": "What language is spoken, is the speaker a man or a woman, and what is the message about?"},
    ]},
])
print(answer)

The speaker is speaking German. He is a man, and the message is an introduction to a new activity: painting animals with closed eyes. He begins by saying, "Today we try something new. We paint animals with closed eyes." He then adds, "My first attempt looks like a potato with ears," describing his initial result humorously.

answer, _ = generate([
    {"role": "system", "content": "Answer questions about the provided audio clip. Base every answer only on what can be heard."},
    {"role": "user", "content": [
        {"type": "audio", "audio": "german.wav"},
        {"type": "text", "text": "What activity does the speaker suggest?\nA. Cooking a new recipe\nB. Drawing animals with closed eyes\nC. Planting flowers\nD. Learning a dance\nPlease choose the answer from the options."},
    ]},
])
print(answer)

Drawing animals with closed eyes

from datetime import date, timedelta
from urllib.request import Request, urlopen
from PIL import Image

tools = [{
    "name": "book_table",
    "description": "Book a table at a restaurant using its name and city.",
    "parameters": {
        "type": "object",
        "properties": {
            "restaurant_name": {"type": "string"},
            "city": {"type": "string"},
            "date": {"type": "string", "description": "Local date, YYYY-MM-DD."},
            "time": {"type": "string", "description": "Local time, HH:MM in 24-hour format."},
            "party_size": {"type": "integer", "minimum": 1},
            "reservation_name": {"type": "string"},
        },
        "required": ["restaurant_name", "city", "date", "time", "party_size", "reservation_name"],
    },
}]

photo_url = "https://upload.wikimedia.org/wikipedia/commons/8/80/Cafe_de_Flore_Paris_France.jpg"
with urlopen(Request(photo_url, headers={"User-Agent": "LFM3Cookbook"}), timeout=30) as response:
    Image.open(response).convert("RGB").save("restaurant.jpg")

# Record the spoken request
booking_date = date.today() + timedelta(days=7)
_, spoken_request = generate([
    {"role": "system", "content": "Perform TTS in British English using a male voice."},
    {"role": "user", "content": f"Please book a table for two at this restaurant in Paris on {booking_date:%B %d, %Y} at seven in the evening, under Alex Taylor."},
], generation_mode="audio")
sf.write("request.wav", spoken_request, SR)

inputs = processor.apply_chat_template(
    [
        {"role": "system", "content": "Use the tools to help the user complete their goal."},
        {"role": "user", "content": [
            {"type": "image", "image": "restaurant.jpg"},
            {"type": "audio", "audio": "request.wav"},
        ]},
    ],
    tools=tools, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt",
).to(model.device)
with torch.inference_mode():
    out = model.generate(**inputs, max_new_tokens=512, generation_mode="text")
print(processor.decode(out.sequences[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))

<tool_call>
<function=book_table>
<parameter=restaurant_name>
Café de Flore
</parameter>
<parameter=city>
Paris
</parameter>
<parameter=date>
2026-11-01
</parameter>
<parameter=time>
19:00
</parameter>
<parameter=party_size>
2
</parameter>
<parameter=reservation_name>
Alex Taylor
</parameter>
</function>
</tool_call>

conversation = [
    {"role": "user", "content": [{"type": "image", "image": "bee.jpg"}, {"type": "text", "text": "Describe this picture in one sentence."}]},
]
for mode in ["text", "interleaved"]:
    torch.manual_seed(0)
    reply, speech = generate(conversation, generation_mode=mode)
    print(f"{mode}: {reply}")
    if speech is not None:
        sf.write(f"{mode}.wav", speech, SR)

text: A bright pink cosmos flower, with a black-and-yellow bumblebee perched on its yellow center, is the sharp focal point of this close-up, surrounded by softly blurred pink blooms, dried brown flower heads, a single vivid red flower, and large green leaves in the background.
interleaved: A bright pink cosmos flower with a fuzzy bumblebee perched on its yellow center is the focal point, surrounded by other pink blooms, a small red flower, and blurred green leaves in the background.