nswer is satisfactory, or 'NO' if it is not.",

},"role": "user",

"content": f"Question: {question}\nExpected Answer: {expected_answer}\nGenerated Answer: {generated_answer}\nIs the generated answer relevant and accurate?",

},

],

model="gpt-3.5-turbo",

judgment = chat_completion.choices[0].message.content.strip().lower()

return judgment == "yes"

retrieval_scores = []

generation_scores = []

for item in eval_data:

retrieval_score = evaluate_retrieval(

item["question"], item["expected_answer"], corpus

retrieval_scores.append(retrieval_score)

generated_answer = answer_question(item["question"], corpus)

generation_score = evaluate_generation(

item["question"], item["expected_answer"], generated_answer

generation_scores.append(generation_score)

retrieval_accuracy = sum(retrieval_scores) / len(retrieval_scores)

generation_accuracy = sum(generation_scores) / len(generation_scores)

print(f"Retrieval Accuracy: {retrieval_accuracy:.2f}")

print(f"Generation Accuracy: {generation_accuracy:.2f}")

As you can see, we've added two evaluation functions: evaluate_retrieval and evaluate_generation. The evaluate_retrieval function checks if the retrieved chunks contain any words from the expected answer. The evaluate_generation function uses OpenAI's chat completion LLM to evaluate the quality of the generated answer.

We then loop through the evaluation data, which contains questions and expected answers, and evaluate the retrieval and generation components of our RAG pipeline. Finally, we calculate the accuracy of both components and print the results:

As you can see, we get 100% accuracy for both retrieval and generation in this example. Not bad! The sections that follow will provide a more detailed and sophisticated implementation of RAG evaluation, but this example shows how you can think about it at a high level!

PreviousEvaluation and metrics

NextRetrieval evaluation

Last updated 19 days ago