Document Loader dan Text Splitter

from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter

# 1. Load PDF document
loader = PyPDFLoader("data/company_policy.pdf")
docs = loader.load()
print(f"Total pages: {len(docs)}")

# 2. Split documents into chunks
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=["\n\n", "\n", " ", ""]
)
splits = splitter.split_documents(docs)
print(f"Total chunks: {len(splits)}")

# 3. Check a chunk
print(docs[0].page_content[:200])
print(splits[0].page_content)
print(splits[0].metadata)

from langchain_community.document_loaders import WebBaseLoader
from langchain_text_splitters import CharacterTextSplitter

# 1. Load web pages
loader = WebBaseLoader(["https://example.com/docs"])
docs = loader.load()
print(f"Loaded {len(docs)} documents")

# 2. Split into chunks
splitter = CharacterTextSplitter(
    chunk_size=500,
    chunk_overlap=50
)
splits = splitter.split_documents(docs)
print(f"Total chunks: {len(splits)}")

# 3. View chunk
print(splits[0].page_content)
print(splits[0].metadata)