d1-omni-2B

pip install "transformers>=5.19" torch torchvision pillow

import torch
from transformers import AutoModel

device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
dtype = torch.float32 if device == "cpu" else torch.bfloat16
model = AutoModel.from_pretrained("LiquidAI/d1-omni-2B", trust_remote_code=True, dtype=dtype).to(device)

questions = {
    "refund": {"type": "noul", "instructions": "Is the customer asking for a refund?"},
    "team": {"type": "choice", "instructions": "Which team should handle this?",
             "criteria": {"billing": "Charges, refunds, invoices", "shipping": "Delivery and parcels",
                          "technical": "App or site faults"}},
    "topics": {"type": "choice", "multiple": True, "instructions": "Which topics does the message raise?",
               "criteria": {"billing": "Payments and charges", "delivery": "Shipping and parcels",
                            "login": "Signing in to the account"}},
}
output = model.system_one("I was charged twice and my parcel never arrived. Please refund one charge.", questions)
a = output["answers"]
print(round(a["refund"]["noul"], 2), a["team"]["choice"], a["topics"]["choices"])
# 0.94 billing ['billing', 'delivery']

import numpy as np

# A recording: three short beeps over quiet noise, 16 kHz mono
rate = 16000
clip = np.random.default_rng(0).normal(0, 0.003, 4 * rate).astype(np.float32)
for start in (0.5, 1.5, 2.5):
    clip[int(start * rate):int(start * rate) + 3200] += 0.3 * np.sin(2 * np.pi * 880 * np.arange(3200) / rate)

output = model.system_one(None, {
    "beeps": {"type": "number", "instructions": "How many beeps are there?", "range": [0, 10], "step": 1},
    "speech": {"type": "noul", "instructions": "Does someone speak in the recording?"},
}, media=[{"type": "audio", "audio": clip, "sampling_rate": rate}])
a = output["answers"]
print(a["beeps"]["number"], a["speech"]["noul"] > 0.5)
# 3.0 False

from transformers.image_utils import load_image

image = load_image("http://images.cocodataset.org/val2017/000000039769.jpg")
output = model.system_one(None, {
    "cats": {"type": "number", "instructions": "How many cats are there?", "range": [0, 10], "step": 1},
    "remote": {"type": "box", "instructions": "Where is the remote control on the left?"},
    "every_cat": {"type": "boxes", "instructions": "Every cat.", "max": 4},
}, media=[image])
a = output["answers"]
print(a["cats"]["number"], a["every_cat"]["count"], {k: round(v, 2) for k, v in a["remote"]["box"].items()})
# 2.0 2 {'top': 0.15, 'bottom': 0.24, 'left': 0.06, 'right': 0.27}