pip install "openllm[vllm]" llama-index
TRUST_REMOTE_CODE=True openllm start microsoft/phi-2 --quantize int8

from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContextfrom llama_index.llms.openllm import OpenLLMfrom llama_index.text_splitter import SentenceSplitter
from llama_index.llama_dataset import download_llama_dataset
rag_dataset, documents = download_llama_dataset("PaulGrahamEssayDataset", "./data")
llm = OpenLLMAPI(address="<http://localhost:3000>")
# Break down the document into manageable chunks (each of size 1024 characters, with a 20-character overlap)text_splitter = SentenceSplitter(chunk_size=1024, chunk_overlap=20)
# Create a ServiceContext with the custom model and all the configurationsservice_context = ServiceContext.from_defaults(llm=llm,embed_model="local",text_splitter=text_splitter,context_window=8192,num_output=4096,)
# Build an index over the documents using the customized LLM in the ServiceContextindex = VectorStoreIndex.from_documents(documents, service_context=service_context,,vector_store_kwargs={"deep memory": True})
# Query your data using the built indexquery_engine = index.as_query_engine()response = query_engine.query("Who is Paul Graham?")print(response)