LangSmith

from langsmith import traceable
from langchain_sambanova import ChatSambaNova

llm = ChatSambaNova(model="MiniMax-M3", max_tokens=2000)

@traceable(name="sambanova-capabilities-check")
def ask(question: str) -> str:
    return llm.invoke(question).content

print(ask("What can you do?"))

# Expected output
I can answer questions, provide explanations, generate text, and help with a wide range of tasks.

from langsmith import Client
from langchain_sambanova import ChatSambaNova
from pydantic import BaseModel, Field

ls = Client()
MODEL = "MiniMax-M3"
DATASET = "sambanova-qa"

llm = ChatSambaNova(model=MODEL, max_tokens=2000)

if not ls.has_dataset(dataset_name=DATASET):
    dataset = ls.create_dataset(DATASET)
    ls.create_examples(dataset_id=dataset.id, examples=[
        {"inputs": {"question": "What is the capital of France?"},
         "outputs": {"answer": "Paris"}},
        {"inputs": {"question": "What is 2+2?"},
         "outputs": {"answer": "4"}},
        {"inputs": {"question": "Who wrote Hamlet?"},
         "outputs": {"answer": "William Shakespeare"}},
    ])

def target(inputs: dict) -> dict:
    return {"answer": llm.invoke(inputs["question"]).content}

class Grade(BaseModel):
    """Grade a submitted answer against a reference answer."""

    correct: bool = Field(description="Whether the submitted answer is factually correct")
    reasoning: str = Field(description="Brief justification for the verdict")

judge = ChatSambaNova(model=MODEL, max_tokens=1000).bind_tools(
    [Grade], tool_choice="required"
)

def correctness(outputs: dict, reference_outputs: dict) -> dict:
    result = judge.invoke(
        f"Reference answer: {reference_outputs['answer']}\n"
        f"Submitted answer: {outputs['answer']}\n"
        "Is the submitted answer factually correct?"
    )
    args = result.tool_calls[0]["args"]
    return {"key": "correctness", "score": bool(args["correct"]),
            "comment": args["reasoning"]}

results = ls.evaluate(
    target,
    data=DATASET,
    evaluators=[correctness],
    experiment_prefix="sambanova-minimax-m3",
    max_concurrency=2,
)
print(results.experiment_name)

# Expected output
View the evaluation results for experiment: 'sambanova-minimax-m3-adb0ee8b' at:
https://smith.langchain.com/o/<org-id>/datasets/<dataset-id>/compare?selectedSessions=<session-id>

sambanova-minimax-m3-adb0ee8b