getpass

### Dataset namedataset_name = "LCEL-QA"

### INDEX
from bs4 import BeautifulSoup as Soupfrom langchain_community.vectorstores import Chromafrom langchain_openai import OpenAIEmbeddingsfrom langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoaderfrom langchain_text_splitters import RecursiveCharacterTextSplitter
# Load docsurl = "https://python.langchain.com/v0.1/docs/expression_language/"loader = RecursiveUrlLoader(url=url, max_depth=20, extractor=lambda x: Soup(x, "html.parser").text)docs = loader.load()
# Split into chunkstext_splitter = RecursiveCharacterTextSplitter(chunk_size=4500, chunk_overlap=200)splits = text_splitter.split_documents(docs)
# Embed and store in Chromavectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings())
# Indexretriever = vectorstore.as_retriever()

### RAG bot
import openaifrom langsmith import traceablefrom langsmith.wrappers import wrap_openai
class RagBot:
def __init__(self, retriever, model: str = "gpt-4-0125-preview"):self._retriever = retriever# Wrapping the client instruments the LLMself._client = wrap_openai(openai.Client())self._model = model
@traceable()def retrieve_docs(self, question):return self._retriever.invoke(question)
@traceable()def invoke_llm(self, question, docs):response = self._client.chat.completions.create(model=self._model,messages=[{"role": "system","content": "You are a helpful AI code assistant with expertise in LCEL."" Use the following docs to produce a concise code solution to the user question.\n\n"f"## Docs\n\n{docs}",},{"role": "user", "content": question},],)
# Evaluators will expect "answer" and "contexts"return {"answer": response.choices[0].message.content,"contexts": [str(doc) for doc in docs],}
@traceable()def get_answer(self, question: str):docs = self.retrieve_docs(question)return self.invoke_llm(question, docs)
rag_bot = RagBot(retriever)

response = rag_bot.get_answer("How to build a RAG chain in LCEL?")response["answer"][:150]

def predict_rag_answer(example: dict):"""Use this for answer evaluation"""response = rag_bot.get_answer(example["input_question"])return {"answer": response["answer"]}
def predict_rag_answer_with_context(example: dict):"""Use this for evaluation of retrieved documents and hallucinations"""response = rag_bot.get_answer(example["input_question"])return {"answer": response["answer"], "contexts": response["contexts"]}

from langchain import hubfrom langchain_openai import ChatOpenAI
# Grade promptgrade_prompt_answer_accuracy = prompt = hub.pull("langchain-ai/rag-answer-vs-reference")
def answer_evaluator(run, example) -> dict:"""A simple evaluator for RAG answer accuracy"""
# Get question, ground truth answer, RAG chain answerinput_question = example.inputs["input_question"]reference = example.outputs["output_answer"]prediction = run.outputs["answer"]
# LLM graderllm = ChatOpenAI(model="gpt-4-turbo", temperature=0)
# Structured promptanswer_grader = grade_prompt_answer_accuracy | llm
# Run evaluatorscore = answer_grader.invoke({"question": input_question,"correct_answer": reference,"student_answer": prediction})score = score["Score"]
return {"key": "answer_v_reference_score", "score": score}

from langsmith.evaluation import evaluate
experiment_results = evaluate(predict_rag_answer,data=dataset_name,evaluators=[answer_evaluator],experiment_prefix="rag-answer-v-reference",metadata={"version": "LCEL context, gpt-4-0125-preview"},)

# Grade promptgrade_prompt_answer_helpfulness = prompt = hub.pull("langchain-ai/rag-answer-helpfulness")
def answer_helpfulness_evaluator(run, example) -> dict:"""A simple evaluator for RAG answer helpfulness"""
# Get question, ground truth answer, RAG chain answerinput_question = example.inputs["input_question"]prediction = run.outputs["answer"]
# LLM graderllm = ChatOpenAI(model="gpt-4-turbo", temperature=0)
# Structured promptanswer_grader = grade_prompt_answer_helpfulness | llm
# Run evaluatorscore = answer_grader.invoke({"question": input_question,"student_answer": prediction})score = score["Score"]
return {"key": "answer_helpfulness_score", "score": score}

experiment_results = evaluate(predict_rag_answer,data=dataset_name,evaluators=[answer_helpfulness_evaluator],experiment_prefix="rag-answer-helpfulness",metadata={"version": "LCEL context, gpt-4-0125-preview"},)

# Promptgrade_prompt_hallucinations = prompt = hub.pull("langchain-ai/rag-answer-hallucination")
def answer_hallucination_evaluator(run, example) -> dict:"""A simple evaluator for generation hallucination"""
# RAG inputsinput_question = example.inputs["input_question"]contexts = run.outputs["contexts"]
# RAG answerprediction = run.outputs["answer"]
# LLM graderllm = ChatOpenAI(model="gpt-4-turbo", temperature=0)
# Structured promptanswer_grader = grade_prompt_hallucinations | llm
# Get scorescore = answer_grader.invoke({"documents": contexts,"student_answer": prediction})score = score["Score"]
return {"key": "answer_hallucination", "score": score}

# Grade promptgrade_prompt_doc_relevance = hub.pull("langchain-ai/rag-document-relevance")
def docs_relevance_evaluator(run, example) -> dict:"""A simple evaluator for document relevance"""
# RAG inputsinput_question = example.inputs["input_question"]contexts = run.outputs["contexts"]
# LLM graderllm = ChatOpenAI(model="gpt-4-turbo", temperature=0)
# Structured promptanswer_grader = grade_prompt_doc_relevance | llm
# Get scorescore = answer_grader.invoke({"question":input_question,"documents":contexts})score = score["Score"]
return {"key": "document_relevance", "score": score}

experiment_results = evaluate(predict_rag_answer_with_context,data=dataset_name,evaluators=[docs_relevance_evaluator],experiment_prefix="rag-doc-relevance",metadata={"version": "LCEL context, gpt-4-0125-preview"},)

from langsmith.schemas import Example, Runfrom langsmith.evaluation import evaluate
def document_relevance_grader(root_run: Run, example: Example) -> dict:"""A simple evaluator that checks to see if retrieved documents are relevant to the question"""
# Get specific steps in our RAG pipeline, which are noted with @traceable decoratorrag_pipeline_run = next(run for run in root_run.child_runs if run.name == "get_answer")retrieve_run = next(run for run in rag_pipeline_run.child_runs if run.name == "retrieve_docs")contexts = "\n\n".join(doc.page_content for doc in retrieve_run.outputs["output"])input_question = example.inputs["input_question"]
# LLM graderllm = ChatOpenAI(model="gpt-4-turbo", temperature=0)
# Structured promptanswer_grader = grade_prompt_doc_relevance | llm
# Get scorescore = answer_grader.invoke({"question":input_question,"documents":contexts})score = score["Score"]
return {"key": "document_relevance", "score": score}
def answer_hallucination_grader(root_run: Run, example: Example) -> dict:"""A simple evaluator that checks to see the answer is grounded in the documents"""
# RAG inputrag_pipeline_run = next(run for run in root_run.child_runs if run.name == "get_answer")retrieve_run = next(run for run in rag_pipeline_run.child_runs if run.name == "retrieve_docs")contexts = "\n\n".join(doc.page_content for doc in retrieve_run.outputs["output"])
# RAG outputprediction = rag_pipeline_run.outputs["answer"]
# LLM graderllm = ChatOpenAI(model="gpt-4-turbo", temperature=0)
# Structured promptanswer_grader = grade_prompt_hallucinations | llm
# Get scorescore = answer_grader.invoke({"documents": contexts,"student_answer": prediction})score = score["Score"]
return {"key": "answer_hallucination", "score": score}
experiment_results = evaluate(predict_rag_answer,data=dataset_name,evaluators=[document_relevance_grader, answer_hallucination_grader],metadata={"version": "LCEL context, gpt-4-0125-preview"},)