from urllib.parse import urlparse
import requestsfrom bs4 import BeautifulSoup
documents = []for url in urls:try:page = requests.get(url)except Exception:raise ValueError(f"One of the inputs is not a valid url: {url}")
hostname = custom_hostname or urlparse(url).hostname or ""
soup = BeautifulSoup(page.content, "html.parser")
data = ""extra_info = {"title": soup.select_one("title"),"url_source": url,"creation_date": datetime.now().date().isoformat(),# Convert datetime to ISO format string}if hostname in self._website_extractor:data, metadata = self._website_extractor[hostname](soup=soup, url=url, include_url_in_text=include_url_in_text)extra_info.update(metadata)
else:data = soup.getText()
documents.append(Document(text=data, id_=url, extra_info=extra_info))
return documents

def _mpweixin_reader(soup: Any, **kwargs) -> Tuple[str, Dict[str, Any]]:meta_tag_title = soup.find('meta', attrs={'property': 'og:title'})title = meta_tag_title['content']extra_info = {"title": title,}text = soup.select_one("div #page-content").getText()    return text, extra_info

https://r.jina.ai/https://example.com

class JinaWebReader(BasePydanticReader):def __init__(self) -> None:"""Initialize with parameters."""
def load_data(self, urls: List[str]) -> List[Document]:if not isinstance(urls, list):raise ValueError("urls must be a list of strings.")
documents = []for url in urls:new_url = "https://r.jina.ai/" + urlresponse = requests.get(new_url)text = response.text
# Extract Titletitle_match = re.search(r"Title:\s*(.*)", text)title = title_match.group(1) if title_match else None
# Extract URL Sourceurl_match = re.search(r"URL Source:\s*(.*)", text)url_source = url_match.group(1) if url_match else None
# Extract Markdown Contentmarkdown_match = re.search(r"Markdown Content:\s*(.*)", text, re.DOTALL)markdown_content = markdown_match.group(1).strip() if markdown_match else None
# Compose metadatametadata: Dict = {"title": title,"url_source": url_source,"creation_date": datetime.now().date().isoformat(),# Convert datetime to ISO format string}
documents.append(Document(text=markdown_content, id_=url, metadata=metadata or {}))
        return documents