Ng6EdZddlmZddlmZmZmZmZmZm Z m Z m Z m Z m Z mZmZmZddlmZmZmZmZmZmZmZmZGddeZGddeeZGd d eZGd d eZGd deZGddeZGdde Z GddeZ!GddeZ"GddeZ#GddeZ$eeeeegZ%eeeegZ&ee eegZ'ee eegZ(ee e gZ)GddeZ*Gdde*Z+Gdd e*Z,Gd!d"e*Z-Gd#d$e*Z.Gd%d&e*Z/eee"e#e$gZ0eeee!gZ1eeee!gZ2eeee!gZ3Gd'd(e*eZ4Gd)d*e4Z5Gd+d,e4Z6Gd-d.e4Z7Gd/d0e4Z8 d9d6Z9e:d7kr e9d8Sd8S):a Data classes and parser implementations for *incremental* chart parsers, which use dynamic programming to efficiently parse a text. A "chart parser" derives parse trees for a text by iteratively adding "edges" to a "chart". Each "edge" represents a hypothesis about the tree structure for a subsequence of the text. The "chart" is a "blackboard" for composing and combining these hypotheses. A parser is "incremental", if it guarantees that for all i, j where i < j, all edges ending at i are built before any edges ending at j. This is appealing for, say, speech recognizer hypothesis filtering. The main parser class is ``EarleyChartParser``, which is a top-down algorithm, originally formulated by Jay Earley (1970). ) perf_counter) BottomUpPredictCombineRuleBottomUpPredictRuleCachedTopDownPredictRuleChart ChartParserEdgeIEmptyPredictRule"FilteredBottomUpPredictCombineRule!FilteredSingleEdgeFundamentalRuleLeafEdge LeafInitRuleSingleEdgeFundamentalRuleTopDownInitRule)!FeatureBottomUpPredictCombineRuleFeatureBottomUpPredictRule FeatureChartFeatureChartParserFeatureEmptyPredictRule FeatureSingleEdgeFundamentalRuleFeatureTopDownInitRuleFeatureTopDownPredictRulec>eZdZdZdZdZdZdZdZdZ dZ d S) IncrementalChartctd|D|_i|_i|_dS)Nc3K|]}gVdSN.0xs R/var/www/html/ai-engine/env/lib/python3.11/site-packages/nltk/parse/earleychart.py z.IncrementalChart.initialize..@s">>q>>>>>>)tuple _positions _edgelists _edge_to_cpls_indexesselfs r" initializezIncrementalChart.initialize>sA>>DOO,=,=>>>>>  r$cDt|Sr)list iteredgesr*s r"edgeszIncrementalChart.edgesIsDNN$$%%%r$c$d|jDS)Nc3$K|] }|D]}|V dSrr)r edgelistedges r"r#z-IncrementalChart.iteredges..Ms/JJJJJJJJJJJr$)r'r*s r"r/zIncrementalChart.iteredgesLsJJT_JJJJr$c |j|}ikrt|St}t |}||jvr||t fd|D}t|j|||gS)Nc3(K|] }|V dSrr)r key restrictionss r"r#z*IncrementalChart.select..^s(==3\#&======r$r'itersortedkeysr%r) _add_indexgetr+endr8r3 restr_keysvalss ` r"selectzIncrementalChart.selectOs?3' 2  >> !L--//00 :&&  T] * * OOJ ' ' '====*=====DM*-c266tR@@AAAr$c|D])}tt|std|z*td|Dx}|j|<t |jD]V\}}||}|D]Ftfd|D}||g GWdS)NBad restriction: %sc3K|]}iVdSrrrs r"r#z.IncrementalChart._add_index..h"1P1P"1P1P1P1P1P1Pr$c3HK|]}t|VdSrgetattrr r7r4s r"r#z.IncrementalChart._add_index..ns6HHc/WT3//11HHHHHHr$ hasattrr ValueErrorr%r&r) enumerater' setdefaultappend r+rAr7indexr@r3 this_indexrBr4s @r"r=zIncrementalChart._add_indexas > >C5#&& > !6!<=== >-21P1Pdoo>O>O1P1P1P,P,PP j)'t77 = =MCsJ  = =HHHHZHHHHH%%dB//66t<<<< = = =r$c}|jD]O\}}tfd|D}|||gPdS)Nc3HK|]}t|VdSrrIrKs r"r#z:IncrementalChart._register_with_indexes..ts6DD#+s++--DDDDDDr$r@r)itemsr%rPrQr+r4r@rArSrBs ` r"_register_with_indexesz'IncrementalChart._register_with_indexesqshhjj!%!4!4!6!6 9 9 JDDDDDDDDDD #J ! !$ + + 2 24 8 8 8 8 9 9r$cj|j||dSr)r'r@rQ)r+r4s r" _append_edgezIncrementalChart._append_edgews+  #**400000r$cJt|dzS)N)range num_leavesr*s r"r&zIncrementalChart._positionszsT__&&*+++r$N) __name__ __module__ __qualname__r,r0r/rCr=rZr\r&rr$r"rr=s   &&&KKKBBB$=== 999 111,,,,,r$rc eZdZdZdZdZdS)FeatureIncrementalChartc j|}ikrt|St}t |}|jvr|t fd|D}tj|||gS)Nc3NK|]}|V dSr)_get_type_if_possible)r r7r8r+s r"r#z1FeatureIncrementalChart.select..sG  >AD & &|C'8 9 9      r$r9r?s` ` r"rCzFeatureIncrementalChart.selects?3' 2  >> !L--//00 :&&  T] * * OOJ ' ' '     EO     DM*-c266tR@@AAAr$c|D])}tt|std|z*tdDx}j|<t jD]W\}}||}|D]Gtfd|D}||g HXdS)NrEc3K|]}iVdSrrrs r"r#z5FeatureIncrementalChart._add_index..rGr$c3nK|]/}t|V0dSrrhrJr r7r4r+s r"r#z5FeatureIncrementalChart._add_index..sX../AwtS/A/A/C/CDDr$rLrRs ` @r"r=z"FeatureIncrementalChart._add_indexs > >C5#&& > !6!<=== >-21P1Pdoo>O>O1P1P1P,P,PP j)'t77 = =MCsJ  = =)%%dB//66t<<<<  = = =r$c}jD]P\}}tfd|D}|||gQdS)Nc3nK|]/}t|V0dSrrlrms r"r#zAFeatureIncrementalChart._register_with_indexes..sUEH**+=74+=+=+?+?@@r$rWrYs`` r"rZz.FeatureIncrementalChart._register_with_indexesshhjj!%!4!4!6!6 9 9 JLVD #J ! !$ + + 2 24 8 8 8 8  9 9r$N)rarbrcrCr=rZrr$r"rere~sDBBB(===&99999r$receZdZdZdS)CompleteFundamentalRulec#K|}|||d|D]D}||}||||r|VEdSNT)startr@ is_completelhs)r@rCnextsymmove_dot_forwardinsert_with_backpointer)r+chartgrammar left_edger@ right_edgenew_edges r"_apply_incompletez)CompleteFundamentalRule._apply_incompletesmmoo ,,3Di6G6G6I6I'    J!11*..2B2BCCH,,Xy*MM    r$Nrarbrcrrr$r"rqrqs#     r$rqc(eZdZeZdZdS) CompleterRulec#|Kt|ts$|j|||Ed{VdSdSr isinstancer _fundamental_ruleapplyr+rzr{r4s r"rzCompleterRule.applysX$)) J-33E7DII I I I I I I I I I J Jr$Nrarbrcrqrrrr$r"rr7//11JJJJJr$rc(eZdZeZdZdS) ScannerRulec#|Kt|tr$|j|||Ed{VdSdSrrrs r"rzScannerRule.applysX dH % % J-33E7DII I I I I I I I I I J Jr$Nrrr$r"rrrr$rceZdZdS) PredictorRuleNrarbrcrr$r"rrDr$rceZdZdZdS)FilteredCompleteFundamentalRulec#pK|r||||Ed{VdSdSr)ru_apply_completers r"rz%FilteredCompleteFundamentalRule.applysY      B++E7DAA A A A A A A A A A B Br$N)rarbrcrrr$r"rrs(BBBBBr$rceZdZdZdS)FeatureCompleteFundamentalRulec#K|j}|}|||d|D] }|||||Ed{V!dSrs)rr@rCrwr)r+rzr{r|frr@r}s r"rz0FeatureCompleteFundamentalRule._apply_incompletes  #mmoo ,,3Di6G6G6I6I'   G GJxxw :FF F F F F F F F F G Gr$Nrrr$r"rrs(GGGGGr$rc"eZdZeZdS)FeatureCompleterRuleNrarbrcrrrr$r"rr6688r$rc"eZdZeZdS)FeatureScannerRuleNrrr$r"rrrr$rceZdZdS)FeaturePredictorRuleNrrr$r"rrrr$rc*eZdZdZeddefdZddZdS)IncrementalChartParsera An *incremental* chart parser implementing Jay Earley's parsing algorithm: | For each index end in [0, 1, ..., N]: | For each edge such that edge.end = end: | If edge is incomplete and edge.next is not a part of speech: | Apply PredictorRule to edge | If edge is incomplete and edge.next is a part of speech: | Apply ScannerRule to edge | If edge is complete: | Apply CompleterRule to edge | Return any complete parses in the chart r2c||_||_||_||_g|_g|_|D]\}|jdkr|j|(|jdkr|j|NtddS)a Create a new Earley chart parser, that uses ``grammar`` to parse texts. :type grammar: CFG :param grammar: The grammar used to parse texts. :type trace: int :param trace: The level of tracing that should be used when parsing a text. ``0`` will generate no tracing output; and higher numbers will produce more verbose tracing output. :type trace_chart_width: int :param trace_chart_width: The default total width reserved for the chart in trace output. The remainder of each line will be used to display edges. :param chart_class: The class that should be used to create the charts used by this parser. rr^z9Incremental inference rules must have NUM_EDGES == 0 or 1N) _grammar_trace_trace_chart_width _chart_class_axioms_inference_rules NUM_EDGESrQrN)r+r{strategytracetrace_chart_width chart_classrules r"__init__zIncrementalChartParser.__init__/s4   "3' "  D~"" ##D))))1$$%,,T2222 R   r$Nc ||j}|j}t|}|j|||}|j}|j|dzz}|r"t| ||j D]4}t| ||}||||||5|j } t|dzD]} |dkrtd| dt|| } | r| } | D]g} t| ||| }||| ||||D]/}|| kr| |0h| |S)Nr^z * Processing queue: )r@)r_trace_new_edgesr.rcheck_coveragerrr`printpretty_format_leavesrrrr_rCpopr@rQ)r+tokensrtrace_new_edgesrzr{trace_edge_widthaxiom new_edgesinference_rulesr@agendar4rr~s r" chart_parsez"IncrementalChartParser.chart_parseZs =KE/f $$V,,,!!&))- 2u7G7G7I7IA7MN  @ %,,-=>> ? ? ?\ N NEU[[8899I OE5)U>S00"MM(3334  4 r$r)rarbrc__doc__BU_LC_INCREMENTAL_STRATEGYrrrrr$r"rrsU  $,$ ))))V!!!!!!r$rceZdZdZdS)EarleyChartParserc 8tj||tfi|dSr)rrEARLEY_STRATEGYr+r{ parser_argss r"rzEarleyChartParser.__init__s%'gVV+VVVVVr$Nrarbrcrrr$r"rr~s(WWWWWr$rceZdZdZdS)IncrementalTopDownChartParserc 8tj||tfi|dSr)rrTD_INCREMENTAL_STRATEGYrs r"rz&IncrementalTopDownChartParser.__init__6' '2  6A     r$Nrrr$r"rr#     r$rceZdZdZdS)IncrementalBottomUpChartParserc 8tj||tfi|dSr)rrBU_INCREMENTAL_STRATEGYrs r"rz'IncrementalBottomUpChartParser.__init__rr$Nrrr$r"rrrr$rceZdZdZdS)(IncrementalBottomUpLeftCornerChartParserc 8tj||tfi|dSr)rrrrs r"rz1IncrementalBottomUpLeftCornerChartParser.__init__s6' '5  9D     r$Nrrr$r"rrrr$rceZdZdZdS) IncrementalLeftCornerChartParserc ~|stdtj||tfi|dS)NzNIncrementalLeftCornerParser only works for grammars without empty productions.) is_nonemptyrNrrLC_INCREMENTAL_STRATEGYrs r"rz)IncrementalLeftCornerChartParser.__init__sc""$$ -  ' '2  6A     r$Nrrr$r"rrs#     r$rceZdZedefdZdS)FeatureIncrementalChartParserc 4tj||f|||d|dS)N)rrr)rr)r+r{rrrrs r"rz&FeatureIncrementalChartParser.__init__sH '   /#         r$N)rarbrc"BU_LC_INCREMENTAL_FEATURE_STRATEGYrerrr$r"rrs34+       r$rceZdZdZdS)FeatureEarleyChartParserc 8tj||tfi|dSr)rrEARLEY_FEATURE_STRATEGYrs r"rz!FeatureEarleyChartParser.__init__s6%. '2  6A     r$Nrrr$r"rrrr$rceZdZdZdS)$FeatureIncrementalTopDownChartParserc 8tj||tfi|dSr)rrTD_INCREMENTAL_FEATURE_STRATEGYrs r"rz-FeatureIncrementalTopDownChartParser.__init__6%. ':  >I     r$Nrrr$r"rrrr$rceZdZdZdS)%FeatureIncrementalBottomUpChartParserc 8tj||tfi|dSr)rrBU_INCREMENTAL_FEATURE_STRATEGYrs r"rz.FeatureIncrementalBottomUpChartParser.__init__rr$Nrrr$r"rrrr$rceZdZdZdS)/FeatureIncrementalBottomUpLeftCornerChartParserc 8tj||tfi|dSr)rrrrs r"rz8FeatureIncrementalBottomUpLeftCornerChartParser.__init__s6%. '=  AL     r$Nrrr$r"rrrr$rTF$I saw John with a dog with my cookiecddl}ddl}ddlm}|} |rt dt | t dt ||} t | t t | |} t} | | } t| | }t| z } |rt||ks Jd|r|D]}t |nt dt||rt d | dSdS) z0 A demonstration of the Earley parsers. rN) demo_grammarz * Grammarz * Sentence:)rzNot all parses foundz Nr trees:zTime:) systimenltk.parse.chartrrsplitrrrr.parsesrtlen) print_times print_grammar print_treesrsent numparsesrrrr{rearleytrzrtrees r"demorsJJJKKK------lnnG k g - $KKK ZZ\\F &MMM GGGwe 4 4 4FA   v & &E %,,w}}// 0 0FA@6{{i''')?'''(  D $KKKK  k3v;;''' gqr$__main__N)TFTrrr);rrrrrrrrrr r r r r rrrnltk.parse.featurechartrrrrrrrrrrerqrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrarr$r"rs?                      >,>,>,>,>,u>,>,>,B.9.9.9.9.9. .9.9.9l     7   JJJJJ+JJJJJJJJ)JJJ     ,   BBBBB&GBBB G G G G G%E G G G99999=99999999999     4   LNNOMOOKMMMOO LNNO LNN LNN   LNN&&((##%%\\\\\[\\\~WWWWW.WWW      $:        %;        /E         '=    "LNN LNN""$$ #LNN  ""$$ #LNN%%''""$$ &"     $: