
    Ng                    X    d dl mZ d dlmZmZ d dlmZ  G d de          ZddddZdS )    )annotations)AnyList)TextSplitterc                  :     e Zd ZdZ	 	 	 dddd fdZddZ xZS )SpacyTextSplitteraR  Splitting text using Spacy package.


    Per default, Spacy's `en_core_web_sm` model is used and
    its default max_length is 1000000 (it is the length of maximum character
    this model takes which can be increased for large files). For a faster, but
    potentially less accurate splitting, you can use `pipeline='sentencizer'`.
    

en_core_web_sm@B T)strip_whitespace	separatorstrpipeline
max_lengthintr   boolkwargsr   returnNonec                    t                      j        di | t          ||          | _        || _        || _        dS )z#Initialize the spacy text splitter.r   N )super__init__"_make_spacy_pipeline_for_splitting
_tokenizer
_separator_strip_whitespace)selfr   r   r   r   r   	__class__s         Z/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_text_splitters/spacy.pyr   zSpacyTextSplitter.__init__   sT     	""6"""<
 
 
 $!1    text	List[str]c                      fd                      |          j        D             }                     | j                  S )z&Split incoming text and return chunks.c              3  B   K   | ]}j         r|j        n|j        V  d S )N)r   r#   text_with_ws).0sr   s     r!   	<genexpr>z/SpacyTextSplitter.split_text.<locals>.<genexpr>%   sG       
 
 ,@AFF!.
 
 
 
 
 
r"   )r   sents_merge_splitsr   )r   r#   splitss   `  r!   
split_textzSpacyTextSplitter.split_text#   sT    
 
 
 
__T**0
 
 
 !!&$/:::r"   )r	   r
   r   )r   r   r   r   r   r   r   r   r   r   r   r   )r#   r   r   r$   )__name__
__module____qualname____doc__r   r.   __classcell__)r    s   @r!   r   r      sz           (#	2 "&2 2 2 2 2 2 2 2"; ; ; ; ; ; ; ;r"   r   r   r   r   r   r   r   r   r   c                   	 dd l }n# t          $ r t          d          w xY w| dk    r&ddlm}  |            }|                    d           n |                    | ddg          }||_        |S )Nr   zCSpacy is not installed, please install it with `pip install spacy`.sentencizer)Englishnertagger)exclude)spacyImportErrorspacy.lang.enr6   add_pipeloadr   )r   r   r:   r6   r5   s        r!   r   r   ,   s    
 
 
 
Q
 
 	

 =  ))))))"799]++++jjE83DjEE!+s    !N)r   r   r   r   r   r   )	
__future__r   typingr   r   langchain_text_splitters.baser   r   r   r   r"   r!   <module>rB      s    " " " " " "         6 6 6 6 6 6!; !; !; !; !; !; !; !;J )2       r"   