
    Ng                     d    d dl mZmZmZmZ d dlmZ d dlmZm	Z	 d dl
mZ  G d dee          ZdS )    )AnyCallableDictList)Document)	BaseModelmodel_validator)
BaseLoaderc                        e Zd ZU dZeed<   	 eed<   	 eege	f         ed<   	 dedeege	f         f fdZ
 ed          eded	efd
                        Zd	ee	         fdZ xZS )ApifyDatasetLoadera  Load datasets from `Apify` web scraping, crawling, and data extraction platform.

    For details, see https://docs.apify.com/platform/integrations/langchain

    Example:
        .. code-block:: python

            from langchain_community.document_loaders import ApifyDatasetLoader
            from langchain_core.documents import Document

            loader = ApifyDatasetLoader(
                dataset_id="YOUR-DATASET-ID",
                dataset_mapping_function=lambda dataset_item: Document(
                    page_content=dataset_item["text"], metadata={"source": dataset_item["url"]}
                ),
            )
            documents = loader.load()
    apify_client
dataset_iddataset_mapping_functionc                 N    t                                          ||           dS )ar  Initialize the loader with an Apify dataset ID and a mapping function.

        Args:
            dataset_id (str): The ID of the dataset on the Apify platform.
            dataset_mapping_function (Callable): A function that takes a single
                dictionary (an Apify dataset item) and converts it to an instance
                of the Document class.
        )r   r   N)super__init__)selfr   r   	__class__s      n/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/document_loaders/apify_dataset.pyr   zApifyDatasetLoader.__init__%   s6     	!<T 	 	
 	
 	
 	
 	
    before)modevaluesreturnc                     	 ddl m}  |            }t          |j        d          x}r|j        dxx         dz  cc<   ||d<   n# t
          $ r t          d          w xY w|S )zYValidate environment.

        Args:
            values: The values to validate.
        r   )ApifyClienthttpx_clientz
user-agentz; Origin/langchainr   z`Could not import apify-client Python package. Please install it with `pip install apify-client`.)r   r   getattrhttp_clientheadersImportError)clsr   r   clientr   s        r   validate_environmentz'ApifyDatasetLoader.validate_environment4   s    	000000 []]F&v'9>JJJ| K$\2226JJ222%+F>"" 	 	 	E  	 s   AA Ac                     | j                             | j                                      d          j        }t          t          | j        |                    S )zLoad documents.T)clean)r   datasetr   
list_itemsitemslistmapr   )r   dataset_itemss     r   loadzApifyDatasetLoader.loadM   sP     %%do66AAAMMS 	 C5}EEFFFr   )__name__
__module____qualname____doc__r   __annotations__strr   r   r   r   r	   classmethodr$   r   r-   __classcell__)r   s   @r   r   r   	   s         & TOOO6&vx'78888>

9A4&(BR9S
 
 
 
 
 
 _(###$ 3    [ $#.Gd8n G G G G G G G Gr   r   N)typingr   r   r   r   langchain_core.documentsr   pydanticr   r	   )langchain_community.document_loaders.baser
   r    r   r   <module>r;      s    , , , , , , , , , , , , - - - - - - / / / / / / / / @ @ @ @ @ @IG IG IG IG IGY IG IG IG IG IGr   