pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain

import bs4from langchain_community.document_loaders import WebBaseLoaderloader = WebBaseLoader(web_paths=("https://data.eastmoney.com/report/zw_stock.jshtml?encodeUrl=35LdZWW1ERIe7AWLlyVZFHptPNDUH2qN4gEfzUIhYsc=", "https://data.eastmoney.com/report/zw_stock.jshtml?encodeUrl=OPw8X34UDWQq6g0u70KgHW6e5Ad8C5kc3TYV6t9BZsw=", "https://data.eastmoney.com/report/zw_stock.jshtml?encodeUrl=OPw8X34UDWQq6g0u70KgHbx8/qX6gdD4f6j3/4IEWIA="),bs_kwargs=dict(parse_only=bs4.SoupStrainer(class_=("detail-header", "newsContent")# 根据网页信息不同调整需要保留的字段)),)blog_docs = loader.load()# 数据读取

# Splitfrom langchain.text_splitter import RecursiveCharacterTextSplittertext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=512,chunk_overlap=128)
# Make splitssplits = text_splitter.split_documents(blog_docs)

# Indexfrom langchain_openai import OpenAIEmbeddingsfrom langchain_community.vectorstores import Chromavectorstore = Chroma.from_documents(documents=splits,embedding=OpenAIEmbeddings())

retriever = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 6}))
# 测试一下检索的片段retrieved_docs = retriever.invoke("东方财富营收是多少")len(retrieved_docs)

from langchain import hubfrom langchain_openai import ChatOpenAI
prompt = hub.pull("rlm/rag-prompt")# LLMllm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0.01)

from langchain_core.output_parsers import StrOutputParserfrom langchain_core.runnables import RunnablePassthrough
# 文本片段组合def format_docs(docs):return "\n\n".join(doc.page_content for doc in docs)

rag_chain = ({"context": retriever | format_docs, "question": RunnablePassthrough()}| prompt| llm| StrOutputParser())
for chunk in rag_chain.stream("东方财富的营收是多少"):    print(chunk, end="", flush=True)