!pip install llama-index
# My OpenAI Keyimport osos.environ['OPENAI_API_KEY'] = ""
import loggingimport sysimport requests
logging.basicConfig(stream=sys.stdout, level=logging.INFO)logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
from llama_index import VectorStoreIndex, SimpleDirectoryReaderfrom IPython.display import Markdown, display
# download paul graham's essayresponse = requests.get("<https://www.dropbox.com/s/f6bmb19xdg0xedm/paul_graham_essay.txt?dl=1>")essay_txt = response.textwith open("pg_essay.txt", "w") as fp:fp.write(essay_txt)
# load documentsdocuments = SimpleDirectoryReader(input_files=['pg_essay.txt']).load_data()
index = VectorStoreIndex.from_documents(documents)
# set Logging to DEBUG for more detailed outputsquery_engine = index.as_query_engine(similarity_top_k=2)
response = query_engine.query("What did the author do growing up?",)
print(response.source_nodes[0].node.get_text())

# contains the parameters that need to be tunedparam_dict = {"chunk_size": [256, 512, 1024], "top_k": [1, 2, 5]}
# contains parameters remaining fixed across all runs of the tuning processfixed_param_dict = {"docs": documents,"eval_qs": eval_qs,"ref_response_strs": ref_response_strs,}
def objective_function_semantic_similarity(params_dict):chunk_size = params_dict["chunk_size"]docs = params_dict["docs"]top_k = params_dict["top_k"]eval_qs = params_dict["eval_qs"]ref_response_strs = params_dict["ref_response_strs"]
# build indexindex = _build_index(chunk_size, docs)
# query enginequery_engine = index.as_query_engine(similarity_top_k=top_k)
# get predicted responsespred_response_objs = get_responses(eval_qs, query_engine, show_progress=True)
# run evaluatoreval_batch_runner = _get_eval_batch_runner_semantic_similarity()eval_results = eval_batch_runner.evaluate_responses(eval_qs, responses=pred_response_objs, reference=ref_response_strs)
# get semantic similarity metricmean_score = np.array([r.score for r in eval_results["semantic_similarity"]]).mean()
return RunResult(score=mean_score, params=params_dict)
param_tuner = ParamTuner(param_fn=objective_function_semantic_similarity,param_dict=param_dict,fixed_param_dict=fixed_param_dict,show_progress=True,)
results = param_tuner.tune()

import osfrom llama_index.postprocessor.cohere_rerank import CohereRerank
api_key = os.environ["COHERE_API_KEY"]cohere_rerank = CohereRerank(api_key=api_key, top_n=2) # return top 2 nodes from reranker
query_engine = index.as_query_engine(similarity_top_k=10, # we can set a high top_k here to ensure maximum relevant retrievalnode_postprocessors=[cohere_rerank], # pass the reranker to node_postprocessors)
response = query_engine.query("What did Elon Musk do?",)

from llama_index.core.query_engine import RetrieverQueryEnginefrom llama_index.core.response_synthesizers import CompactAndRefinefrom llama_index.postprocessor.longllmlingua import LongLLMLinguaPostprocessorfrom llama_index.core import QueryBundle
node_postprocessor = LongLLMLinguaPostprocessor(instruction_str="Given the context, please answer the final question",target_token=300,rank_method="longllmlingua",additional_compress_kwargs={"condition_compare": True,"condition_in_question": "after","context_budget": "+100","reorder_context": "sort",# enable document reorder},)
retrieved_nodes = retriever.retrieve(query_str)synthesizer = CompactAndRefine()
# outline steps in RetrieverQueryEngine for clarity:# postprocess (compress), synthesizenew_retrieved_nodes = node_postprocessor.postprocess_nodes(retrieved_nodes, query_bundle=QueryBundle(query_str=query_str))
print("\\n\\n".join([n.get_content() for n in new_retrieved_nodes]))
response = synthesizer.synthesize(query_str, new_retrieved_nodes)

from llama_index.core.postprocessor import LongContextReorder
reorder = LongContextReorder()
reorder_engine = index.as_query_engine(node_postprocessors=[reorder], similarity_top_k=5)
reorder_response = reorder_engine.query("Did the author meet Sam Altman?")

from llama_index.core import VectorStoreIndex, SimpleDirectoryReaderfrom llama_index.core.output_parsers import LangchainOutputParserfrom llama_index.llms.openai import OpenAIfrom langchain.output_parsers import StructuredOutputParser, ResponseSchema
# load documents, build indexdocuments = SimpleDirectoryReader("../paul_graham_essay/data").load_data()index = VectorStoreIndex.from_documents(documents)
# define output schemaresponse_schemas = [ResponseSchema(name="Education",description="Describes the author's educational experience/background.",),ResponseSchema(name="Work",description="Describes the author's work experience/background.",),]
# define output parserlc_output_parser = StructuredOutputParser.from_response_schemas(response_schemas)output_parser = LangchainOutputParser(lc_output_parser)
# Attach output parser to LLMllm = OpenAI(output_parser=output_parser)
# obtain a structured responsequery_engine = index.as_query_engine(llm=llm)response = query_engine.query("What are a few things the author did growing up?",)print(str(response))

from pydantic import BaseModelfrom typing import List
from llama_index.program.openai import OpenAIPydanticProgram
# Define output schema (without docstring)class Song(BaseModel):title: strlength_seconds: int
class Album(BaseModel):name: strartist: strsongs: List[Song]
# Define openai pydantic programprompt_template_str = """\\Generate an example album, with an artist and a list of songs. \\Using the movie {movie_name} as inspiration.\\"""program = OpenAIPydanticProgram.from_defaults(output_cls=Album, prompt_template_str=prompt_template_str, verbose=True)
# Run program to get structured outputoutput = program(movie_name="The Shining", description="Data model for an album.")

# load datadocuments = SimpleDirectoryReader(input_dir="./data/source_files").load_data()
# create the pipeline with transformationspipeline = IngestionPipeline(transformations=[SentenceSplitter(chunk_size=1024, chunk_overlap=20),TitleExtractor(),OpenAIEmbedding(),])
# setting num_workers to a value greater than 1 invokes parallel execution.nodes = pipeline.run(documents=documents, num_workers=4)