Ng" ddlZddlZddlmZmZmZmZmZmZm Z m Z m Z ddl m Z ddlmZddlmZdedefdZd ed edefd Zd ed edee eddffdZdede eeffdZGddeZdS)N) AnyCallableDict GeneratorIterableIteratorListOptionalTuple)urlparse)Document) WebBaseLoadercontentreturncDt|SN)strget_text)rs h/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/document_loaders/sitemap.py_default_parsing_functionrs w!! " ""meta_contentcd|di|S)Nsourceloc)rrs r_default_meta_functionrs d5k *T **riterablesizec#Kt|}ttj||x}r*|Vttj||x}(dSdSr)iterlist itertoolsislice)rr ititems r _batch_blockr(st hBy'D1122 2$ y'D1122 2$rurlc<t|}|j|jfS)zExtract the scheme + domain from a given URL. Args: url (str): The input URL. Returns: return a 2-tuple of scheme and domain )r schemenetloc)r) parsed_uris r_extract_scheme_and_domainr.#s #J  j/ //rceZdZdZ ddedeeed eed eed ed eed e de de dede ffd Z ddde dedee fdZ deefdZxZS) SitemapLoaderaHLoad a sitemap and its URLs. **Security Note**: This loader can be used to load all URLs specified in a sitemap. If a malicious actor gets access to the sitemap, they could force the server to load URLs from other domains by modifying the sitemap. This could lead to server-side request forgery (SSRF) attacks; e.g., with the attacker forcing the server to load URLs from internal service endpoints that are not publicly accessible. While the attacker may not immediately gain access to this data, this data could leak into downstream systems (e.g., data loader is used to load data for indexing). This loader is a crawler and web crawlers should generally NOT be deployed with network access to any internal servers. Control access to who can submit crawling requests and what network access the crawler has. By default, the loader will only load URLs from the same domain as the sitemap if the site map is not a local file. This can be disabled by setting restrict_to_same_domain to False (not recommended). If the site map is a local file, no such risk mitigation is applied by default. Use the filter URLs argument to limit which URLs can be loaded. See https://python.langchain.com/docs/security NrFT web_path filter_urlsparsing_function blocksizeblocknum meta_functionis_localcontinue_on_failurerestrict_to_same_domain max_depthkwargsc  x||dkrtd|dkrtd ddl} n#t$rtdwxYwtjdd|gi| ||_| |_|pt|_|pt|_ ||_ ||_ ||_ ||_| |_dS) aInitialize with webpage path and optional filter URLs. Args: web_path: url of the sitemap. can also be a local path filter_urls: a list of regexes. If specified, only URLS that match one of the filter URLs will be loaded. *WARNING* The filter URLs are interpreted as regular expressions. Remember to escape special characters if you do not want them to be interpreted as regular expression syntax. For example, `.` appears frequently in URLs and should be escaped if you want to match a literal `.` rather than any character. restrict_to_same_domain takes precedence over filter_urls when restrict_to_same_domain is True and the sitemap is not a local file. parsing_function: Function to parse bs4.Soup output blocksize: number of sitemap locations per block blocknum: the number of the block that should be loaded - zero indexed. Default: 0 meta_function: Function to parse bs4.Soup output for metadata remember when setting this method to also copy metadata["loc"] to metadata["source"] if you are using this field is_local: whether the sitemap is a local file. Default: False continue_on_failure: whether to continue loading the sitemap if an error occurs loading a url, emitting a warning instead of raising an exception. Setting this to True makes the loader more robust, but also may result in missing data. Default: False restrict_to_same_domain: whether to restrict loading to URLs to the same domain as the sitemap. Attention: This is only applied if the sitemap is not a local file! max_depth: maximum depth to follow sitemap links. Default: 10 Nz&Sitemap blocksize should be at least 1rz(Sitemap blocknum can not be lower then 0zAlxml package not found, please install it with `pip install lxml` web_pathsr) ValueErrorlxml ImportErrorsuper__init__allow_url_patternsr:rr4rr7r5r6r8r9r;)selfr2r3r4r5r6r7r8r9r:r;r<rA __class__s rrDzSitemapLoader.__init__MsZ  Y]]EFF F a<<GHH H  KKKK   S   88H:8888 #.'>$ 0 M4M*D.D"    #6 "s 4AdepthsouprIrc ||jkrgSg}|dD] d}|s|j|jr-|js&tt|jkrg|j r!tfd|j Ds|  fddD|dD]i}|d}|s| |jgdd}| |||d z j|S) zParse sitemap xml and load into a list of dicts. Args: soup: BeautifulSoup object. depth: current depth of the sitemap. Default: 0 Returns: List of dicts. r)rc3BK|]}tj|VdSr)rematch).0regexp_patternloc_texts r z.SitemapLoader.parse_sitemap..sC33"22333333rcNi|]!}|x|j"Sr)findtext)rOtagpropr)s r z/SitemapLoader.parse_sitemap..sB # -r)rlastmod changefreqprioritysitemapxmlrr>rH)r;find_allrTrUstripr:r8r.r2rEanyappend scrape_allextend parse_sitemap) rFrJrIelsrr\ soup_childrQrWr)s @@@rrdzSitemapLoader.parse_sitemaps DN " "I==''  C((5//C x~~''H+ DM -h77;UM<<& s3333&*&=33300  JJK    }}Y// H HG,,u%%C #(U;;A>J JJt))*EAI)FF G G G G rc#K|jrN ddl}n#t$rtdwxYwt|j}||d}n||jd}||}|j[tt||j}t|}|dz |j krtd||j }|d|D}t|D]E\}} t!|| ||||  VFdS) z Load sitemap.rNzUbeautifulsoup4 package not found, please install it with `pip install beautifulsoup4`r])parserr>zBSelected sitemap does not contain enough blocks for given blocknumcJg|] }d|v|d!S)r)r_)rOels r z+SitemapLoader.lazy_load..s+"R"R"Rerkk2e9??#4#4kkkr) page_contentmetadata)r8bs4rBopenr2 BeautifulSoup_scraperdr5r#r(lenr6r@rb enumerater r4r7) rFrnfprJreelblocks blockcountresultsiresults r lazy_loadzSitemapLoader.lazy_loads = =     !9  dm$$B$$R//DD<< e<< %Ldn==>>HXJA~ -- Xt}-//"R"Rs"R"R"RSS"7++  IAv!226::++CFF;;      s*) NNNrNFFTr1)__name__ __module__ __qualname____doc__rr r rintboolrrDdictrdrr rz __classcell__)rGs@rr0r00s\>,0/3#',0$)(,H#H#H#d3i(H##8, H# C= H#  H# )H#H#"H#"&H#H#H#H#H#H#H#H#T89222#22T$Z2222h!8H-!!!!!!!!rr0)r$rMtypingrrrrrrr r r urllib.parser langchain_core.documentsr -langchain_community.document_loaders.web_baserrrrrrr(r.r0rrrrs                       "!!!!!------GGGGGG#s#s####+++++++839T$Zt=S3T 0C 0E#s(O 0 0 0 0|||||M|||||r