NgddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlZddlmZddlmZdd lmZdd lm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*dd l+m,Z,e rddl-m.Z.ej/e0Z1Gdde,Z2GddeeZ3Gdde3Z4Gdde3Z5Gdde3Z6GddeZ7Gdde3Z8Gdde3Z9Gdd e3Z:Gd!d"e3Z;Gd#d$e3Z<Gd%d&e3Z=Gd'd(e"Z>Gd)d*e3Z?Gd+d,e3Z@e5ZAdS)-N)ABC)StringIO)Path) TYPE_CHECKINGAnyDictIteratorListMappingOptionalSequenceUnion)urlparse)Document)get_from_dict_or_env) BaseLoader)Blob)DedocBaseLoader)AmazonTextractPDFParserDocumentIntelligenceParserPDFMinerParserPDFPlumberParser PyMuPDFParserPyPDFium2Parser PyPDFParser)UnstructuredFileLoader)TextLinearizationConfigceZdZdZdefdZdS)UnstructuredPDFLoaderaLoad `PDF` files using `Unstructured`. You can run the loader in one of two modes: "single" and "elements". If you use "single" mode, the document will be returned as a single langchain Document object. If you use "elements" mode, the unstructured library will split the document into elements such as Title and NarrativeText. You can pass in additional unstructured kwargs after mode to apply different unstructured settings. Examples -------- from langchain_community.document_loaders import UnstructuredPDFLoader loader = UnstructuredPDFLoader( "example.pdf", mode="elements", strategy="fast", ) docs = loader.load() References ---------- https://unstructured-io.github.io/unstructured/bricks.html#partition-pdf returnc6ddlm}|dd|ji|jS)Nr) partition_pdffilename)unstructured.partition.pdfr" file_pathunstructured_kwargs)selfr"s d/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/document_loaders/pdf.py _get_elementsz#UnstructuredPDFLoader._get_elementsGs4<<<<<<}QQdnQ8PQQQN)__name__ __module__ __qualname____doc__r r*r$r+r)rr/s@.RtRRRRRRr+rceZdZdZdddeeefdeefdZ ddZ e d ede fd Z e d ede fd Ze d ede fd Zedefd ZdS) BasePDFLoaderzBase Loader class for `PDF` files. If the file is a web path, it will download it to a temporary file, use it, then clean up the temporary file after completion. Nheadersr&r3ct||_d|_||_d|jvr)tj|j|_tj|js||jr{tj |_ tj |j\}}| |jr2t|jjdd}tj|j jd|}|j|_||jst'j|j|j}|jdkrt-d|jzt/|d 5}||jdddn #1swxYwYt||_dSdStj|jst-d |jzdS) zInitialize with a file path. Args: file_path: Either a local, S3 or web path to a PDF file. headers: Headers to use for GET request to download a file from a web path. N~/tmpr2z3Check the url of your file; returned status code %swb)modez'File path %s is not a valid file or url)strr&web_pathr3ospath expanduserisfile _is_valid_urltempfileTemporaryDirectorytemp_dirsplitext_is_s3_presigned_urlrsplitjoinname _is_s3_urlrequestsget status_code ValueErroropenwritecontent)r(r&r3_suffixtemp_pdfrfs r)__init__zBasePDFLoader.__init__Ts7Y  $. W//??DNw~~dn-- Y$2D2DT^2T2T Y$799DM((88IAv((88 F!$.116<22 /LFFF=C''$M-( (...'!GGAI&&&'''''''''''''''!$X / /// YFWXX X Y YsG--G14G1r c\t|dr|jdSdS)NrE)hasattrrEcleanupr(s r)__del__zBasePDFLoader.__del__ws8 4 $ $ $ M ! ! # # # # # $ $r+urlcpt|}t|jot|jS)zCheck if the url is valid.)rboolnetlocscheme)r^parseds r)rBzBasePDFLoader._is_valid_url{s.#FM"":tFM':'::r+cp t|}|jdkr |jrdSdS#t$rYdSwxYw)zcheck if the url is S3s3TF)rrbrarOr^results r)rKzBasePDFLoader._is_s3_urlsR c]]F}$$$t5   55 s !' 55c t|}ttjd|jS#t $rYdSwxYw)z'Check if the url is a presigned S3 url.z\.s3\.amazonaws\.com$F)rr`researchrarOrfs r)rGz"BasePDFLoader._is_s3_presigned_urlsQ c]]F ":FMJJKK K   55 s58 AAc,|j|jn|jSN)r=r&r\s r)sourcezBasePDFLoader.sources $ 9t}}t~Mr+)r N)r,r-r.r/rr<rr rrXr] staticmethodr`rBrKrGpropertyrmr$r+r)r1r1Ms3 RV!Y!Y!Y%T "2!Y!Y!Y!Y!YF$$$$;3;4;;;\; \#$\NNNNXNNNr+r1c*eZdZdZdeefdZdS)OnlinePDFLoaderzLoad online `PDF`.r cltt|j}|S)zLoad documents.)rr<r&load)r(loaders r)rszOnlinePDFLoader.loads'&s4>':':;;{{}}r+N)r,r-r.r/r rrsr$r+r)rqrqs9d8nr+rqceZdZdZ dddddedeeeefdeed e d ed eed dffd Z d e e fdZ xZS) PyPDFLoaderuc PyPDFLoader document loader integration Setup: Install ``langchain-community``. .. code-block:: bash pip install -U langchain-community Instantiate: .. code-block:: python from langchain_community.document_loaders import PyPDFLoader loader = PyPDFLoader( file_path = "./example_data/layout-parser-paper.pdf", password = "my-password", extract_images = True, # headers = None # extraction_mode = "plain", # extraction_kwargs = None, ) Lazy load: .. code-block:: python docs = [] docs_lazy = loader.lazy_load() # async variant: # docs_lazy = await loader.alazy_load() for doc in docs_lazy: docs.append(doc) print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python LayoutParser : A Unified Toolkit for Deep Learning Based Document Image Analysis Zejiang Shen1( ), R {'source': './example_data/layout-parser-paper.pdf', 'page': 0} Async load: .. code-block:: python docs = await loader.aload() print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python LayoutParser : A Unified Toolkit for Deep Learning Based Document Image Analysis Zejiang Shen1( ), R {'source': './example_data/layout-parser-paper.pdf', 'page': 0} NFplain)extraction_modeextraction_kwargsr&passwordr3extract_imagesrxryr c ddl}n#t$rtdwxYwt||t |||||_dS)Initialize with a file path.rNzCpypdf package not found, please install it with `pip install pypdf`r2)rzr{rxry)pypdf ImportErrorsuperrXrparser) r(r&rzr3r{rxryr~ __class__s r)rXzPyPDFLoader.__init__s  LLLL   U   G444!)+/     "c# K|jrAtjt|jd|j}ntj|j}|j|Ed{VdSLazy load given path as pages.rbr?N r=r from_datarPr&read from_pathrparser(blobs r) lazy_loadzPyPDFLoader.lazy_load = 2>$t~t"<"<"A"A"C"C$-XXXDD>$.11D;$$T***********r+)NNF)r,r-r.r/r<r rbytesrr`rXr rr __classcell__rs@r)rvrvs::~15"&$   ',0    5e,- $     $D>        2+ ( ++++++++r+rvc\eZdZdZddddedeedeffdZd e e fd Z xZ S) PyPDFium2Loaderz;Load `PDF` using `pypdfium2` and chunks at character level.NFr3r{r&r3r{cxt||t||_dS)r}r2r{N)rrXrr)r(r&r3r{rs r)rXzPyPDFium2Loader.__init__s8 G444%^DDD r+r c# K|jrAtjt|jd|j}ntj|j}|j|Ed{VdSrrrs r)rzPyPDFium2Loader.lazy_loadrr+ r,r-r.r/r<r rr`rXr rrrrs@r)rrsEE #'$ E E E E$ E  E E E E E E+ ( ++++++++r+rc eZdZdZ ddeeefdedededed ef d Ze ded efd Z d e e fd Z dS)PyPDFDirectoryLoaderzLoad a directory with `PDF` files using `pypdf` and chunks at character level. Loader also stores page numbers in metadata. **/[!.]*.pdfFr?glob silent_errors load_hidden recursiver{cZ||_||_||_||_||_||_dSrl)r?rrrrr{)r(r?rrrrr{s r)rXzPyPDFDirectoryLoader.__init__#s7  &"*,r+r c@td|jD S)Nc3@K|]}|dVdS).N startswith).0parts r) z3PyPDFDirectoryLoader._is_visible..5s.CCts++CCCCCCr+)anypartsrs r) _is_visiblez PyPDFDirectoryLoader._is_visible3s$CC CCCCCCCr+ct|j}g}|jr||jn||j}|D]}|r|||s|jr tt||j }| }|D]}t||j d<||#t$r.}|jrt"|n|Yd}~d}~wwxYw|S)Nrrm)rr?rrglobris_filer relative_torrvr<r{rsmetadataextend Exceptionrloggerwarning) r(pdocsitemsirtsub_docsdoces r)rszPyPDFDirectoryLoader.load7sO OO&*nK """!&&:K:K $ $Ayy{{ $##AMM!$4$455 $9I $ $!,SVVDDW!X!X!X#);;==#+< > > > > > >   -   G44444rr c#DKddlm}ddlm}ddlm}t }||jd5}|||d|dd d d n #1swxYwYd |j|jn|ji}t| | Vd S) Load file.rr)LAParams) open_filenamerhtml)codeclaparams output_typeNrm page_contentr) rrpdfminer.layoutrpdfminer.utilsrrr&r=rgetvalue)r(rrr output_stringfprs r)rz!PDFMinerPDFasHTMLLoader.lazy_loads$::::::,,,,,,000000 ]4>4 0 0 B  !"                      (=dnn4= M$:$:$<$ 5 5 5 5 5 5Q8H-QQQQQQQQr+rc eZdZdZddddedeededed df fd Z ded e e fd Z ded e e fd Zd e e fd ZxZS) PyMuPDFLoaderz!Load `PDF` files using `PyMuPDF`.NFrr&r3r{kwargsr c  ddl}n#t$rtdwxYwt||||_||_dS)r}rNzI`PyMuPDF` package not found, please install it with `pip install pymupdf`r2)fitzrrrXr{ text_kwargs)r(r&r3r{rrrs r)rXzPyMuPDFLoader.__init__sz  KKKK   (   G444,!rc+K|rtd|di|j|}t||j}|jrAt jt|j d |j}nt j |j }| |Ed{VdS)NzReceived runtime arguments zd. Passing runtime args to `load` is deprecated. Please pass arguments during initialization instead.)rr{rr) rrrrr{r=rrrPr&rr lazy_parse)r(rrrrs r) _lazy_loadzPyMuPDFLoader._lazy_loads   NNXfXXX    5)4V4 #D4G    = 2>$t~t"<"<"A"A"C"C$-XXXDD>$.11D$$T***********r+c 6t|jdi|S)Nr$)listr)r(rs r)rszPyMuPDFLoader.loads"ODO--f--...r+c#>K|Ed{VdSrl)rr\s r)rzPyMuPDFLoader.lazy_loads.??$$$$$$$$$$$r+)r,r-r.r/r<r rr`rrXr rrr rsrrrs@r)rrs++ #'$ """"$ "  "  " """"""(+3+8H+=++++"/S/T(^////%8H-%%%%%%%%r+rc.eZdZdZ ddededed ed eeee fd e d dffd Z e d eeeffdZ e d efdZ e d efdZd efdZded dfdZded efdZeded efdZd eefdZxZS)MathpixPDFLoaderz)Load `PDF` files using `Mathpix` service.mdFNr&processed_file_formatmax_wait_time_secondsshould_clean_pdfextra_request_datarr c ,t|dd|_t|dd|_|dd|ddt j|fi|||_||ni|_||_||_ dS)aInitialize with a file path. Args: file_path: a file for loading. processed_file_format: a format of the processed file. Default is "md". max_wait_time_seconds: a maximum time to wait for the response from the server. Default is 500. should_clean_pdf: a flag to clean the PDF file. Default is False. extra_request_data: Additional request data. **kwargs: additional keyword arguments. mathpix_api_keyMATHPIX_API_KEYmathpix_api_idMATHPIX_API_IDN) rrrpoprrXrrrr)r(r&rrrrrrs r)rXzMathpixPDFLoader.__init__s( 4 %'8  3 $&6    $d+++ #T***--f---%:""4"@  b &;" 0r+c |j|jdS)N)app_idapp_key)rrr\s r)_mathpix_headersz!MathpixPDFLoader._mathpix_headerss-$:NOOOr+cdS)Nzhttps://api.mathpix.com/v3/pdfr$r\s r)r^zMathpixPDFLoader.urls//r+cRd|jdii|j}dtj|iS)Nconversion_formatsT options_json)rrjsondumps)r(optionss r)datazMathpixPDFLoader.datas: !4#=t"D %  7 3 344r+cLt|jd5}d|i}tj|j|j||j}dddn #1swxYwY|}d|vrtd|dd|vr |d}|Std)Nrfile)r3filesrerrorzMathpix request failed: pdf_idzUnable to send PDF to Mathpix.) rPr&rLpostr^rrrrO)r(rWrresponse response_datars r)send_pdfzMathpixPDFLoader.send_pdfs $.$ ' ' 1QKE}$"7u49H                !  m # #P g8NPPQQ Q } $ $"8,FM=>> >s,AAArc|jdz|z}td|jdD]}tj||j}|}|dd}|dd}|%d|}||d |d d z }t||d d} | d krdS| dkrtdtd| dtj dt)zmWait for processing to complete. Args: pdf_id: a PDF id. Returns: None r6rr2rN error_infoz%Unable to retrieve PDF from Mathpix: z (id)status completedz#Unable to retrieve PDF from MathpixzStatus: z$, waiting for processing to complete) r^rangerrLrMrrrOprinttimesleep TimeoutError) r(rr^rSr r rr error_msgrs r)wait_for_processingz$MathpixPDFLoader.wait_for_processings4hnv%q$4a88  A|C1FGGGH$MMOOM"%%gt44E&**<>>J KEKK )!9j&6!9!9!99I +++"&&x66F$$7"" !FGGGMMMMNNN 1 r+c|||jd|d|j}tj||j}|jdS)Nr6rr2zutf-8)rr^rrLrMrrRdecode)r(rr^r s r)get_processed_pdfz"MathpixPDFLoader.get_processed_pdf9sd   (((AAFAAT%?AA<T-BCCC&&w///r+contentscbdd|dD}|dddd}|ddd d d d d d}|S)ziClean the PDF file. Args: contents: a PDF file contents. Returns:  c<g|]}|d|S)z![]r)rlines r) z.MathpixPDFLoader.clean_pdf..Js) Q Q Qd$//%:P:P QT Q Q Qr+z \section{z# }rz\$$z\%%z\((z\)r)rIrHreplace)rs r) clean_pdfzMathpixPDFLoader.clean_pdf?s99 Q QhnnT22 Q Q Q  ##L$77??RHH   UC ( ( WUC WUC WUC  r+c|}||}|jr||}|j|j|d}t ||gS)N)rmr&rr)r rrr(rmr)r(rrrs r)rszMathpixPDFLoader.loadWsg))&11   0~~h//H"k vVVhBBBCCr+)rrFN)r,r-r.r/r<intr`r rrrXrorr^dictrr rrrnr(r rrsrrs@r)rrs33 &*%(!&7; %1%1%1 #%1 # %1  %1 %T#s(^4 %1%1 %1%1%1%1%1%1NP$sCx.PPPXP0S000X05d555X5 ?# ? ? ? ?###$####J000000 CC\.Dd8nDDDDDDDDr+rc eZdZdZ d dedeeeefdedee ded df fd Z d e e fd Z xZS) PDFPlumberLoaderz$Load `PDF` files using `pdfplumber`.NFr&rdeduper3r{r c ddl}n#t$rtdwxYwt|||pi|_||_||_dS)r}rNzMpdfplumber package not found, please install it with `pip install pdfplumber`r2) pdfplumberrrrXrr.r{)r(r&rr.r3r{r0rs r)rXzPDFPlumberLoader.__init__cs         +   G444&," ,rc0t|j|j|j}|jrAt jt|jd |j}nt j |j}| |S)r)rr.r{rr) rrr.r{r=rrrPr&rrr)r(rrs r)rszPDFPlumberLoader.loadys"(;.    = 2>$t~t"<"<"A"A"C"C$-XXXDD>$.11D||D!!!r+)NFNF)r,r-r.r/r<r r rr`rrXr rrsrrs@r)r-r-`s.. 48"&$ ---gc3h/0- - $ -  - ------, "d8n " " " " " " " "r+r-c eZdZdZ ddddedeeedeedeedeed eed eed ed d dffdZ d e e fdZ d e e fdZeded efdZxZS)AmazonTextractPDFLoaderaLoad `PDF` files from a local file system, HTTP or S3. To authenticate, the AWS client uses the following methods to automatically load credentials: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html If a specific credential profile should be used, you must pass the name of the profile from the ~/.aws/credentials file that is to be used. Make sure the credentials / roles used have the required policies to access the Amazon Textract service. Example: .. code-block:: python from langchain_community.document_loaders import AmazonTextractPDFLoader loader = AmazonTextractPDFLoader( file_path="s3://pdfs/myfile.pdf" ) document = loader.load() N)linearization_configr&textract_featuresclientcredentials_profile_name region_name endpoint_urlr3r4rr ct|| ddln#t$rtdwxYw|rfd|D} ng} |s|s|r ddl} || |} n| } i} |r|| d<|r|| d<| jd i| }n=#t$rtd t$r} td | | d} ~ wwxYwt| || |_ dS)aUInitialize the loader. Args: file_path: A file, url or s3 path for input file textract_features: Features to be used for extraction, each feature should be passed as a str that conforms to the enum `Textract_Features`, see `amazon-textract-caller` pkg client: boto3 textract client (Optional) credentials_profile_name: AWS profile name, if not default (Optional) region_name: AWS region, eg us-east-1 (Optional) endpoint_url: endpoint url for the textract service (Optional) linearization_config: Config to be used for linearization of the output should be an instance of TextLinearizationConfig from the `textractor` pkg r2rNztCould not import amazon-textract-caller python package. Please install it with `pip install amazon-textract-caller`.c*g|]}j|Sr$)Textract_Features)rxtcs r)r"z4AmazonTextractPDFLoader.__init__..s!KKKA,Q/KKKr+) profile_namer8r9textractzRCould not import boto3 python package. Please install it with `pip install boto3`.zCould not load credentials to authenticate with AWS client. Please check that credentials in the specified profile name are valid. )r5r6r4)r@) rrXtextractcallerrboto3Sessionr6rrOrr)r(r&r5r6r7r8r9r3r4featuresrBsession client_paramsrr>rs @r)rXz AmazonTextractPDFLoader.__init__s6 G444  ' ' ' ' '   O    KKKK9JKKKHHH # { l   +7#mm9QmRRGG$mmooG " ?3>M-0A4@M.1'DDmDD   !B    3/033   .&!5    s",A#AB22"C,C''C,cDt|SzLoad given path as pages.rrr\s r)rszAmazonTextractPDFLoader.loadDNN$$%%%r+c#VK|jr0||jrt|j}nOtj|j}t |dkrtd|jd|j |Ed{VdS)zLazy load documentsrz the file z is a multi-page document, but not stored on S3. Textract requires multi-page documents to be on S3.N) r=rKrrr&r3_get_number_of_pagesrOr?rrrs r)rz!AmazonTextractPDFLoader.lazy_loads = T__T];; T]+++DD>$.11D&;;DAAAEE I III ;$$T***********r+rc& ddl}ddlm}m}n#t$rt dwxYw|jdkrW|5}||}t|j cdddS#1swxYwYdS|jdkrXd}| | }t| |D] \}}|dz } |S|jdvrdStd|j) Nr)Image ImageSequencezcCould not import pypdf or Pilloe python package. Please install it with `pip install pypdf Pillow`.zapplication/pdfz image/tiffrL)z image/pngz image/jpegzunsupported mime type: )r~PILrOrPrmimetype as_bytes_io PdfReaderlenpagesrPas_bytes enumerater rO) rr~rOrPinput_pdf_file pdf_reader num_pagesimgrSs r)rMz,AmazonTextractPDFLoader._get_number_of_pagess  LLL 0 0 0 0 0 0 0 0 0   E   =- - -!!## -~"__^<< :+,, - - - - - - - - - - - - - - - - - -]l * *I**T]]__--C!-"8"8"="=>>  1Q   ]9 9 91Ft}FFGG Gs ) )BBB)NNNNNN)r,r-r.r/r<r r rrrXr rrsr rrnrr*rMrrs@r)r3r3sw06: $26%)&*"&J EIJ J J J $HSM2J  J #+3- J c] J smJ $J ''@AJ  J J J J J J X&d8n&&&&+ ( ++++*H4HCHHH\HHHHHr+r3ceZdZdZdefdZdS)DedocPDFLoadera DedocPDFLoader document loader integration to load PDF files using `dedoc`. The file loader can automatically detect the correctness of a textual layer in the PDF document. Note that `__init__` method supports parameters that differ from ones of DedocBaseLoader. Setup: Install ``dedoc`` package. .. code-block:: bash pip install -U dedoc Instantiate: .. code-block:: python from langchain_community.document_loaders import DedocPDFLoader loader = DedocPDFLoader( file_path="example.pdf", # split=..., # with_tables=..., # pdf_with_text_layer=..., # pages=..., # ... ) Load: .. code-block:: python docs = loader.load() print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python Some text { 'file_name': 'example.pdf', 'file_type': 'application/pdf', # ... } Lazy load: .. code-block:: python docs = [] docs_lazy = loader.lazy_load() for doc in docs_lazy: docs.append(doc) print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python Some text { 'file_name': 'example.pdf', 'file_type': 'application/pdf', # ... } Parameters used for document parsing via `dedoc` (https://dedoc.readthedocs.io/en/latest/parameters/pdf_handling.html): with_attachments: enable attached files extraction recursion_deep_attachments: recursion level for attached files extraction, works only when with_attachments==True pdf_with_text_layer: type of handler for parsing, available options ["true", "false", "tabby", "auto", "auto_tabby" (default)] language: language of the document for PDF without a textual layer, available options ["eng", "rus", "rus+eng" (default)], the list of languages can be extended, please see https://dedoc.readthedocs.io/en/latest/tutorials/add_new_language.html pages: page slice to define the reading range for parsing is_one_column_document: detect number of columns for PDF without a textual layer, available options ["true", "false", "auto" (default)] document_orientation: fix document orientation (90, 180, 270 degrees) for PDF without a textual layer, available options ["auto" (default), "no_change"] need_header_footer_analysis: remove headers and footers from the output result need_binarization: clean pages background (binarize) for PDF without a textual layer need_pdf_table_analysis: parse tables for PDF without a textual layer r cHddlm}||j|j|jS)Nr)make_manager_pdf_config)r&parsing_paramsrH)dedoc.utils.langchainr`r&parsing_parametersrH)r(r`s r) _make_configzDedocPDFLoader._make_configvs>AAAAAA&&n2*    r+N)r,r-r.r/r+rdr$r+r)r^r^s<UUn d      r+r^c zeZdZdZ d dedededeeddf fd Zde e fd Z de e fd Z xZS) DocumentIntelligenceLoaderz+Load a PDF with Azure Document Intelligenceprebuilt-documentNr&r6modelr3r czt|||_t||dS)a Initialize the object for file processing with Azure Document Intelligence (formerly Form Recognizer). This constructor initializes a DocumentIntelligenceParser object to be used for parsing files using the Azure Document Intelligence API. The load method generates a Document node including metadata (source blob and page number) for each page. Parameters: ----------- file_path : str The path to the file that needs to be parsed. client: Any A DocumentAnalysisClient to perform the analysis of the blob model : str The model name or ID to be used for form recognition in Azure. Examples: --------- >>> obj = DocumentIntelligenceLoader( ... file_path="path/to/file", ... client=client, ... model="prebuilt-document" ... ) )r6rhr2N)rrrrX)r(r&r6rhr3rs r)rXz#DocumentIntelligenceLoader.__init__s=D1eLLL  G44444r+cDt|SrHrIr\s r)rszDocumentIntelligenceLoader.loadrJr+c#|Ktj|j}|j|Ed{VdS)rN)rrr&rrrs r)rz$DocumentIntelligenceLoader.lazy_loadsG~dn--;$$T***********r+)rgN)r,r-r.r/r<rr rrXr rrsr rrrs@r)rfrfs55 )"& #5#5#5#5 #5 $ #5  #5#5#5#5#5#5J&d8n&&&&+ ( ++++++++r+rfc `eZdZdZ d deeefdededdffd Zde e fd Z xZ S) ZeroxPDFLoadera Document loader utilizing Zerox library: https://github.com/getomni-ai/zerox Zerox converts PDF document to serties of images (page-wise) and uses vision-capable LLM model to generate Markdown representation. Zerox utilizes anyc operations. Therefore when using this loader inside Jupyter Notebook (or any environment running async) you will need to: ```python import nest_asyncio nest_asyncio.apply() ``` gpt-4o-minir&rh zerox_kwargsr Nc jt| ||_||_dS)N)r&)rrXrorh)r(r&rhrors r)rXzZeroxPDFLoader.__init__s: 9--- &) r+c#0Kddl}ddlm}||d|j|jd|j}t|jdkrB|jdj }|jD]*}t|j |j |j |dV)dSdS) z Loads documnts from pdf utilizing zerox library: https://github.com/getomni-ai/zerox Returns: Iterator[Document]: An iterator over parsed Document instances. rN)zerox)r&rhr7)rmpager[rr$) asynciopyzeroxrrrunr&rhrorUrVrsrrRrm)r(rtrr zerox_outputr[rss r)rzZeroxPDFLoader.lazy_loads !!!!!!{{ E RDN$* R R@Q R R  |! " "Q & &$*2.3I$*  !%"&+ $ %. ' &  r+)rn) r,r-r.r/rr<rrrXr rrrrs@r)rmrms&#d#   :8H-r+rm)Brloggingr>rirCrabcriorpathlibrtypingrrrr r r r r r urllib.parserrLlangchain_core.documentsrlangchain_core.utilsr)langchain_community.document_loaders.baser1langchain_community.document_loaders.blob_loadersr*langchain_community.document_loaders.dedocr0langchain_community.document_loaders.parsers.pdfrrrrrrr1langchain_community.document_loaders.unstructuredr)textractor.data.text_linearization_configr getLogger__file__rrr1rqrvrrrrrrr-r3r^rfrmPagedPDFSplitterr$r+r)rs                         "!!!!!------555555@@@@@@BBBBBBFFFFFFUTTTTTRQQQQQQ  8 $ $RRRRR2RRR