
    Ng                     V    d dl mZmZmZ d dlZd dlmZ d dlmZ  G d de          Z	dS )    )IteratorListUnionN)Document)
BaseLoaderc                   ^    e Zd ZdZ	 d
dedeeee         f         defdZde	e
         fdZd	S )BrowserlessLoaderz3Load webpages with `Browserless` /content endpoint.T	api_tokenurlstext_contentc                 4    || _         	 || _        	 || _        dS )z0Initialize with API token and the URLs to scrapeN)r
   r   r   )selfr
   r   r   s       l/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/document_loaders/browserless.py__init__zBrowserlessLoader.__init__   s&     #$	%(    returnc              #   t  K   | j         D ]}| j        rjt          j        dd| j        i|ddigd          }t          |                                d         d         d	         d         d
         d|i          V  st          j        dd| j        id|i          }t          |j        d|i          V  dS )zLazy load Documents from URLs.z$https://chrome.browserless.io/scrapetokenselectorbody)urlelements)paramsjsondatar   resultstextsource)page_contentmetadataz%https://chrome.browserless.io/contentr   N)r   r   requestspostr
   r   r   r   )r   r   responses      r   	lazy_loadzBrowserlessLoader.lazy_load   s4      9 &	 &	C  %#=:  # !+F%    !)!8!;I!Fq!I&!Q #       $=; s   !) #      C&	 &	r   N)T)__name__
__module____qualname____doc__strr   r   boolr   r   r   r$    r   r   r	   r	   	   s        == QU) ))$)#tCy.$9)IM) ) ) ))8H- ) ) ) ) ) )r   r	   )
typingr   r   r   r!   langchain_core.documentsr   )langchain_community.document_loaders.baser   r	   r+   r   r   <module>r/      s    ( ( ( ( ( ( ( ( ( (  - - - - - - @ @ @ @ @ @6 6 6 6 6
 6 6 6 6 6r   