from langchain_core.documents import Document from langchain_text_splitters import RecursiveCharacterTextSplitter source = Document( page_content=( "checkout incidents page the on-call engineer immediately " "checkout incidents use priority P1 and send status updates every fifteen minutes " "billing questions use priority P3 and route to finance support " "password reset requests use priority P4 and route to the help desk" ), metadata={"source": "support-runbook"}, ) splitter = RecursiveCharacterTextSplitter( chunk_size=75, chunk_overlap=22, separators=["\n\n", "\n", " ", ""], add_start_index=True, ) chunks = splitter.split_documents([source]) print(f"chunk_count={len(chunks)}") for number, chunk in enumerate(chunks, start=1): previous = chunks[number - 2] if number > 1 else None if previous is None: overlap = "n/a" else: previous_end = previous.metadata["start_index"] + len(previous.page_content) overlap = previous_end - chunk.metadata["start_index"] print( f"chunk_{number}: chars={len(chunk.page_content)} " f"start={chunk.metadata['start_index']} " f"overlap_from_previous={overlap} " f"source={chunk.metadata['source']}" ) print(chunk.page_content) assert len(chunks) == 5 assert all(len(chunk.page_content) <= 75 for chunk in chunks) assert all("start_index" in chunk.metadata for chunk in chunks) assert all( chunks[i - 1].metadata["start_index"] + len(chunks[i - 1].page_content) > chunks[i].metadata["start_index"] for i in range(1, len(chunks)) )