import bs4
import requests
from langchain_core.documents import Document


def load_web_page(url: str, bs_kwargs: dict | None = None) -> list[Document]:
    response = requests.get(url, timeout=20)
    response.raise_for_status()
    soup = bs4.BeautifulSoup(response.text, "html.parser", **(bs_kwargs or {}))
    text = soup.get_text("\n", strip=True)
    return [Document(page_content=text, metadata={"source": url})]


page_url = "https://lilianweng.github.io/posts/2023-06-23-agent/"
main_content = bs4.SoupStrainer(class_=("post-title", "post-header", "post-content"))
docs = load_web_page(page_url, bs_kwargs={"parse_only": main_content})
doc = docs[0]

print(f"documents: {len(docs)}")
print(f"source: {doc.metadata['source']}")
print(f"characters: {len(doc.page_content)}")
print("excerpt:")
print(doc.page_content[:320])