Ng^-ddlmZddlZddlZddlmZmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlmZddlmZGddeZGd d ZGd d ZdS) ) annotationsN)BytesIOStringIO)AnyDictIterableListOptionalTuple TypedDictcastDocument)RecursiveCharacterTextSplitterc<eZdZUdZded<ded<ded<ded<dS) ElementTypezElement type as typed dict.strurlxpathcontentzDict[str, str]metadataN)__name__ __module__ __qualname____doc____annotations__Y/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_text_splitters/html.pyrrs?%% HHHJJJLLLrrc>eZdZdZ dddZdd ZddZddZddZdS)HTMLHeaderTextSplitterzU Splitting HTML files based on specified headers. Requires lxml package. Fheaders_to_split_onList[Tuple[str, str]]return_each_elementboolc<||_t||_dS)ayCreate a new HTMLHeaderTextSplitter. Args: headers_to_split_on: list of tuples of headers we want to track mapped to (arbitrary) keys for metadata. Allowed header values: h1, h2, h3, h4, h5, h6 e.g. [("h1", "Header 1"), ("h2", "Header 2)]. return_each_element: Return each element w/ associated headers. N)r$sortedr")selfr"r$s r__init__zHTMLHeaderTextSplitter.__init__s#$7 #)*=#>#>   relementsList[ElementType]returnList[Document]cg}|D]Q}|r8|dd|dkr |ddxxd|dzz cc<<||Rd|DS)zCombine elements with common metadata into chunks Args: elements: HTML element content with associated identifying info and metadata rrz cHg|]}t|d|d Srr page_contentrr.0chunks r zGHTMLHeaderTextSplitter.aggregate_elements_to_chunks..Es?    % "2U:=N O O O   r)append)r(r*aggregated_chunkselements raggregate_elements_to_chunksz3HTMLHeaderTextSplitter.aggregate_elements_to_chunks.s02 2 2G! 2%b)*59LLL ""%i000FWY=O4OO0000"((1111  *    rrrkwargsrc ttj|fi|}|t|jS)zSplit HTML from web URL Args: url: web URL **kwargs: Arbitrary additional keyword arguments. These are usually passed to the fetch url content request. )requestsgetsplit_text_from_filerr)r(rr<rs rsplit_text_from_urlz*HTMLHeaderTextSplitter.split_text_from_urlJs9 L ' ' ' '((););<<Unable to import lxml, please install with `pip install lxml`.Nzutf-8)encodingz!xsl/html_chunks_with_headers.xsltcg|] }|d S)rr)r5headers rr7z?HTMLHeaderTextSplitter.split_text_from_file..xsJJJvJJJrhzhttp://www.w3.org/1999/xhtmlz*//*z*[@class='headers']z*[@class='chunk']c g|] }|jpd SrTrCr5nodes rr7z?HTMLHeaderTextSplitter.split_text_from_file...$(!% Rrz*[@class='xpath']c g|] }|jpd SrVrWrXs rr7z?HTMLHeaderTextSplitter.split_text_from_file..rZrc:i|]}|j|jpdSrV)tagrC)r5rYheader_mappings r z?HTMLHeaderTextSplitter.split_text_from_file..s:"""!%+484dio2"""rc|jvSN)r])x header_filters rz=HTMLHeaderTextSplitter.split_text_from_file..s!%=*@rz*[@class='headers']/*)rrrrcHg|]}t|d|d Sr1rr4s rr7z?HTMLHeaderTextSplitter.split_text_from_file..s?eI&6zARSSSr)lxmlrN ImportError HTMLParserparsepathlibPath__file__parentXSLT fromstringrr"dictfindallr8rjoinfilterr$r;)r(rJrNeparsertree xslt_path xslt_tree transformresult result_domns_mapr*r:rcr^s @@rr@z+HTMLHeaderTextSplitter.split_text_from_file]s   " " " " " " "   P  !!7!33{{4((L**14WW KK ** JJy)) 4%%c&kk22 KJ1IJJJ d67756!))&&99  G455 #::   gg,3OO> >% s *%*N)F)r"r#r$r%)r*r+r,r-)rrr<rr,r-rCrr,r-rJrr,r-) rrrrr)r;rBrHr@rrrr!r!s%*?????"    8 = = = =9999HHHHHHrr!cReZdZdZ dd d Zd!dZd"dZ dd#dZd$dZd%dZ d&dZ dS)'HTMLSectionSplitterz` Splitting HTML files based on specified tag and font sizes. Requires lxml package. Nr"r#rw Optional[str]r<rr,Nonec t||_|9tjtjdz |_n+tj||_||_dS)aCreate a new HTMLSectionSplitter. Args: headers_to_split_on: list of tuples of headers we want to track mapped to (arbitrary) keys for metadata. Allowed header values: h1, h2, h3, h4, h5, h6 e.g. [("h1", "Header 1"), ("h2", "Header 2"]. xslt_path: path to xslt file for document transformation. Uses a default if not passed. Needed for html contents that using different format and layouts. Nzxsl/converting_to_header.xslt) rpr"rjrkrlrmabsoluterwr<)r(r"rwr<s rr)zHTMLSectionSplitter.__init__sm $((;#<#<   X&&-0OOhjj NN%\)44==??DN r documentsIterable[Document]r-cgg}}|D]6}||j||j7|||}t di|j}||S)zSplit documents.) metadatasr)r8r3rcreate_documentsrr<split_documents)r(rtextsrdocresults text_splitters rrz#HTMLSectionSplitter.split_documentssry + +C LL) * * *   S\ * * * *'''CC6EEEE ,,W555rrCrcF|t|SrErFrGs rrHzHTMLSectionSplitter.split_textrIrr List[str]rOptional[List[dict]]c|pigt|z}g}t|D]\}}||D]}tj||}|jD]#} |j| dkr|d|j| <$i||j}t|j|} | | |S)z&Create documents from a list of texts.#TITLE#Titler2) len enumeraterHcopydeepcopyrkeysrr3r8) r(rr _metadatasrirCr6rkeynew_docs rrz$HTMLSectionSplitter.create_documentss32$U"3   '' * *GAt.. * *=A77 >..00@@C~c*i77.6w.?s+9h9%.9"0BXVVV  )))) *rhtml_docList[Dict[str, Optional[str]]]c ddlm}m}n"#t$r}td|d}~wwxYw||d}t |j}g}|dg|z}t|D]\}} | } |dkrd} d} g} n"| j } | j } g} | j D]S}|dzt|kr|||dzkrn+t|tr| |Td |  }|d kr|| || d |S) Nr) BeautifulSoup PageElementzzUnable to import BeautifulSoup/PageElement, please install with `pip install bs4`.z html.parserbodyrh1 rT)rRrtag_name)bs4rrrglistr"rfind_allrrCstripname next_elementsr isinstancerr8rr)r(rrrrtsoupheaderssectionsrrRheader_elementcurrent_headercurrent_header_tagsection_contentr:rs rsplit_html_by_headersz)HTMLSectionSplitter.split_html_by_headerss  6 6 6 6 6 6 6 6 6      }X}55t/44667702--7 233"7++  IAv*0NAvv!*%)"(*!/!4!:!:! !   " " " " " " "   P   !!##{{8L116::KK// JJy)) 46{{s 1,1rJc|}|}|}fd|DS)rLc g|]R}ttt|djt|d|diSS)rrrR)r)rr rr")r5sectionr(s rr7z.7st     S'),--,S1D-E-EF I       r)getvaluerr)r(rJ file_contentrs` rr@z(HTMLSectionSplitter.split_text_from_file-se }} ;;LII --l;;     $    rra)r"r#rwrr<rr,r)rrr,r-r})rrrrr,r-)rrr,r)rrr,rr~) rrrrr)rrHrrrr@rrrrrs$(4 6 6 6 69999CG$****X&      rr) __future__rrrjiorrtypingrrrr r r r r r>langchain_core.documentsr"langchain_text_splitters.characterrrr!rrrrrsA""""""  NNNNNNNNNNNNNNNNNNNN------MMMMMM)NNNNNNNNbY Y Y Y Y Y Y Y Y Y r