RAG文本切分的五个层次1：字符切分基础(实战)

text = "2024年一季度公司实现营业收入24.56亿元（-12.60%），实现归母净利润19.54亿元（-3.70%），扣非后归母净利润19.37亿元（+2.38%），基本每股收益0.12元，（-3.36%）。"
chunks = []chunk_size = 35
for i in range(0, len(text), chunk_size):    chunk = text[i:i + chunk_size]  # 每隔35是一个块    chunks.append(chunk)print(chunks)

from langchain.text_splitter import CharacterTextSplitter

text_splitter = CharacterTextSplitter(chunk_size=35, chunk_overlap=0, separator='', strip_whitespace=False)print(text_splitter.split_text(text))

text_splitter = CharacterTextSplitter(chunk_size=35, chunk_overlap=5, separator='', strip_whitespace=False)st = text_splitter.split_text(text)pprint(st)

text_splitter = CharacterTextSplitter(chunk_size=35, chunk_overlap=0, separator='，', strip_whitespace=False)print(text_splitter.split_text(text))

from llama_index.core.text_splitter import SentenceSplitterfrom llama_index.core import SimpleDirectoryReader

splitter = SentenceSplitter(chunk_size=200,chunk_overlap=15,)

documents = SimpleDirectoryReader(input_files=["./test.txt"]).load_data()

nodes = splitter.get_nodes_from_documents(documents)