DeepEval

import os
from deepeval.models import OpenAIModel
from deepeval.test_case import LLMTestCase
from deepeval.metrics import AnswerRelevancyMetric

model = OpenAIModel(
    model="MiniMax-M3",
    api_key=os.environ["SAMBANOVA_API_KEY"],
    base_url="https://api.sambanova.ai/v1",
)

test_case = LLMTestCase(
    input="What is the capital of France?",
    actual_output="The capital of France is Paris.",
)

metric = AnswerRelevancyMetric(model=model, threshold=0.5)
metric.measure(test_case)
print(metric.score)
print(metric.reason)

1.0
The score is 1.00 because the response directly and accurately addresses the question about the capital of France with no irrelevant information. Excellent work!

import os
from deepeval import assert_test
from deepeval.models import OpenAIModel
from deepeval.test_case import LLMTestCase
from deepeval.metrics import AnswerRelevancyMetric

model = OpenAIModel(
    model="MiniMax-M3",
    api_key=os.environ["SAMBANOVA_API_KEY"],
    base_url="https://api.sambanova.ai/v1",
)

def test_answer_relevancy():
    test_case = LLMTestCase(
        input="What is the capital of France?",
        actual_output="The capital of France is Paris.",
    )
    metric = AnswerRelevancyMetric(model=model, threshold=0.5)
    assert_test(test_case, [metric])

✓ Evaluation completed 🎉! (time taken: 5.44s | token cost: None)
» Test Results (1 total tests):
   » Pass Rate: 100.0% | Passed: 1 | Failed: 0