g[NddlZddlZddlZddlmZmZmZmZddl m Z m Z m Z m Z mZer ddlZddlmZmZer ddlZddlmZmZGdd e Zeed Gd d e ZdS)N)add_end_docstringsis_tf_availableis_torch_availablerequires_backends)ArgumentHandlerDatasetPipelinePipelineExceptionbuild_pipeline_init_args),MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES0MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES)/TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES3TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMESceZdZdZddZdS)%TableQuestionAnsweringArgumentHandlerzB Handles arguments for the TableQuestionAnsweringPipeline Nc t|dddl}|td|]t|tr/|d|d|g}nt|t rt|dkrtd|Dstdd|D|dd|dd|}ntd |d d tt|tst|tj r|Std t|d ||d g}|D]R}t|d|js5|dtd||d|d<S|S)Npandasrz(Keyword argument `table` cannot be None.querytablec3@K|]}t|tVdSN) isinstancedict.0ds k/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/pipelines/table_question_answering.py zATableQuestionAnsweringArgumentHandler.__call__..5s,>>1:a..>>>>>>z:Keyword argument `table` should be a list of dict, but is c34K|]}t|VdSr)typers rr zATableQuestionAnsweringArgumentHandler.__call__..7s-UmUmbcVZ[\V]V]UmUmUmUmUmUmr!zIf keyword argument `table` is a list of dictionaries, each dictionary should have a `table` and `query` key, but only dictionary has keys z `table` and `query` keys.zZInvalid input. Keyword argument `table` should be either of type `dict` or `list`, but is ))rrzTable cannot be None.)rr ValueErrorrrgetlistlenallkeysr types GeneratorTyper# DataFrame)selfrrkwargspdtqa_pipeline_inputstqa_pipeline_inputs r__call__z.TableQuestionAnsweringArgumentHandler.__call__%s2 $))) =GHH H ]%&& 599W+=+=+IeiiX_N`N`Nl',g##E4(( SZZ!^^>>>>>>>$oUmUmglUmUmUmoo8<<((4qg9N9N9Z*/''$vJOPQ(--//vvv$E7)C)C$zRWY^YlGmGm$  )u++))) .3U#C#C"D "5 X X 092<HH X%g.6$%<===.0ll;Mg;V.W.W"7+""r!)NN)__name__ __module__ __qualname____doc__r3r!rrr s2-#-#-#-#-#-#r!rT) has_tokenizercneZdZdZdZeffd ZdZdZfdZ ddZ dd Z dd Z d Z xZS)TableQuestionAnsweringPipelinea Table Question Answering pipeline using a `ModelForTableQuestionAnswering`. This pipeline is only available in PyTorch. Example: ```python >>> from transformers import pipeline >>> oracle = pipeline(model="google/tapas-base-finetuned-wtq") >>> table = { ... "Repository": ["Transformers", "Datasets", "Tokenizers"], ... "Stars": ["36542", "4512", "3934"], ... "Contributors": ["651", "77", "34"], ... "Programming language": ["Python", "Python", "Rust, Python and NodeJS"], ... } >>> oracle(query="How many stars does the transformers repository have?", table=table) {'answer': 'AVERAGE > 36542', 'coordinates': [(0, 1)], 'cells': ['36542'], 'aggregator': 'AVERAGE'} ``` Learn more about the basics of using a pipeline in the [pipeline tutorial](../pipeline_tutorial) This tabular question answering pipeline can currently be loaded from [`pipeline`] using the following task identifier: `"table-question-answering"`. The models that this pipeline can use are models that have been fine-tuned on a tabular question answering task. See the up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=table-question-answering). z table,queryc.tj|i|||_|jdkr.t j}|tn-tj}|t| |tt|j jddo'tt|j jdd|_t!|j jdrdnd|_dS)Ntfaggregation_labelsnum_aggregation_labelstapas)super__init__ _args_parser frameworkrcopyupdaterrrcheck_model_typeboolgetattrmodelconfig aggregatehasattrr#)r. args_parserargsr/mapping __class__s rrBz'TableQuestionAnsweringPipeline.__init__ws$)&)))' >T ! !INPPG NNJ K K K KFKMMG NNG H H H g&&&gdj&79MtTTUU Z^ DJ%'? F F[ [  'tz'8:NOOYGGUY r!c |jdi|S)Nr8)rJ)r.inputss rbatch_inferencez.TableQuestionAnsweringPipeline.batch_inferencestz##F###r!c |jdkrDg}g}d}|djd}|d|j}|d|j}|d|j}d} t |D]h} |W| dddf} t j| } || } t | jdD]} | dddf | }| dddf | dz }| ddd f | dz }|dkr&|dkr |dkrt|||f| | <tj |  tj|j| dddf<|| }|| }|| } ||d|d| d }|j}|jr||j||tj| }|j| tj|jjz}t3jt6t9| D]\} }| dddf | }| dddf | dz }| ddd f | dz }|dkr)|dkr#|dkr||f|fd D}jtjt?|d}|js|fn#|tjt?|dfSg}g}d}|djd}|d}|d}|d}d} t |D]} || dddf} t j| t j } || } t | jdD]} | dddf | }| dddf | dz }| ddd f | dz }|dkr&|dkr |dkrt|||f| | <| | dddf<|| }|| }|| } |t j!|dt j!|dt j!| d }|j}|jr||j||tDj#$tEj%|tDjtEj%|tDjz}t3jt6| } t9tEj| D]\} }| dddf | }| dddf | dz }| ddd f | dz }|dkr)|dkr#|dkr||f|fdD}tEj&t?|d}|js|fn#|tEj&t?|dfS)z Inference used for models that need to process sequences in a sequential fashion, like the SQA models which handle conversational query related to a table. ptN input_idsrattention_masktoken_type_idsrr)rWrXrY)logitscri|]3}|tj|dk4Sg?nparraymeanrkeycoords_to_probss r zGTableQuestionAnsweringPipeline.sequential_inference..<lllUXRXoc.B%C%C%H%H%J%JS%Plllr!)dtype)axiscri|]3}|tj|dk4Sr]r^rbs rrezGTableQuestionAnsweringPipeline.sequential_inference..rfr!)'rDshapetodeviceranger_ zeros_likecpunumpytolistinttorch from_numpyr#longrJ unsqueezer[rLappendlogits_aggregation distributions Bernoulliprobsfloat32 collections defaultdictr' enumeratesqueezecattupleint32 expand_dimsr=mathsigmoidcastconcat)r.rS all_logitsall_aggregations prev_answers batch_sizerWrXrYtoken_type_ids_exampleindexprev_labels_example model_labelsi segment_idcol_idrow_idinput_ids_exampleattention_mask_exampleoutputsr[dist_per_token probabilitiespcolrow logits_batchrds @rsequential_inferencez3TableQuestionAnsweringPipeline.sequential_inferences% >T ! !J! L ,215J{+..t{;;I#$4588EEN#$4588EEN%) "z**. m. m +*@A*F'#%=1D1H1H1J1J1P1P1R1R#S#SL-;E-B*"<#5a#899RR%;AAAqD%A%H%H%J%J1%M !71!=!D!D!F!Fq!IA!M!71!=!D!D!F!Fq!IA!M!Q;;6Q;;:??.1,?O2P.Q.QLO383CL3Q3Q3V3VW\Wa3b3b3e3efjfq3r3r*111a40$-e$4!)7)>&)7)>&**/99!<<#9#C#CA#F#F#9#C#CA#F#F% !>H$++G,FGGG!!&)))!&!4!>!>f!>!M!M . 47M7R7RSXS`7a7a7d7d"(/88! #."9$"?"?%m&;&;&=&=&D&D&F&FGG>>DAq!71!=!D!D!F!Fq!IJ0A6==??BQFC0A6==??BQFCaxxC1HHq'c 3::1===llll\klll  9U:%6%6::L*..sL??|UYW\]mWnWnpqMrMr>s sJ! L ,215J{+I#$45N#$45;;==N%) "z**. m. m +*@A*F'#%=1DBH#U#U#UL-;E-B*"<#5a#899RR%;AAAqD%A%H%H%J%J1%M !71!=!D!D!F!Fq!IA!M!71!=!D!D!F!Fq!IA!M!Q;;6Q;;:??.1,?O2P.Q.QLO3?*111a40$-e$4!)7)>&)7)>&** n->QGGG#%>2Hq#Q#Q#Q#%>2Hq#Q#Q#Q% !>H$++G,FGGG!!&))) " 0K0K L Lrw*BJPP! #."9$"?"?)?&%bj&?&?&E&E&G&G&N&N&P&PQQ>>DAq!71!=!D!D!F!Fq!IJ0A6==??BQFC0A6==??BQFCaxxC1HHq'c 3::1===llll\klll 9U:%6%6::L*..sL??|RYW\]mWnWnpqMrMr>s sr!c|j|i|}tj|fi|}t|dkr|dS|S)a Answers queries according to a table. The pipeline accepts several types of inputs which are detailed below: - `pipeline(table, query)` - `pipeline(table, [query])` - `pipeline(table=table, query=query)` - `pipeline(table=table, query=[query])` - `pipeline({"table": table, "query": query})` - `pipeline({"table": table, "query": [query]})` - `pipeline([{"table": table, "query": query}, {"table": table, "query": query}])` The `table` argument should be a dict or a DataFrame built from that dict, containing the whole table: Example: ```python data = { "actors": ["brad pitt", "leonardo di caprio", "george clooney"], "age": ["56", "45", "59"], "number of movies": ["87", "53", "69"], "date of birth": ["7 february 1967", "10 june 1996", "28 november 1967"], } ``` This dictionary can be passed in as such, or can be converted to a pandas DataFrame: Example: ```python import pandas as pd table = pd.DataFrame.from_dict(data) ``` Args: table (`pd.DataFrame` or `Dict`): Pandas DataFrame or dictionary that will be converted to a DataFrame containing all the table values. See above for an example of dictionary. query (`str` or `List[str]`): Query or list of queries that will be sent to the model alongside the table. sequential (`bool`, *optional*, defaults to `False`): Whether to do inference sequentially or as a batch. Batching is faster, but models like SQA require the inference to be done sequentially to extract relations within sequences, given their conversational nature. padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`): Activates and controls padding. Accepts the following values: - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence if provided). - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different lengths). truncation (`bool`, `str` or [`TapasTruncationStrategy`], *optional*, defaults to `False`): Activates and controls truncation. Accepts the following values: - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum acceptable input length for the model if that argument is not provided. This will truncate row by row, removing rows from the table. - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths greater than the model maximum admissible input size). Return: A dictionary or a list of dictionaries containing results: Each result is a dictionary with the following keys: - **answer** (`str`) -- The answer of the query given the table. If there is an aggregator, the answer will be preceded by `AGGREGATOR >`. - **coordinates** (`List[Tuple[int, int]]`) -- Coordinates of the cells of the answers. - **cells** (`List[str]`) -- List of strings made up of the answer cell values. - **aggregator** (`str`) -- If the model has an aggregator, this returns the aggregator. rr)rCrAr3r()r.rOr/pipeline_inputsresultsrQs rr3z'TableQuestionAnsweringPipeline.__call__ s[V,$+Ti}|||d<|||d<i}|||d<||ifS)Npadding truncation sequentialr8)r.rrrr/preprocess_paramsforward_paramss r_sanitize_parametersz3TableQuestionAnsweringPipeline._sanitize_parameters_sL  +2 i (  !.8 l +  !+5N< ( ."44r!Tc||jdkrd}nd}|d|d}}|jrtd||dkrtd||||j|| }||d<|S) Nr@drop_rows_to_fitdo_not_truncaterrztable is emptyzquery is empty)return_tensorsrr)r#emptyr% tokenizerrD)r.pipeline_inputrrrrrrSs r preprocessz)TableQuestionAnsweringPipeline.preprocessks  yG##/ . %g.w0Gu ; /-.. . =ERKK-.. .uT^Xblsttw r!Fc |d}|jdkr|r|jdi|}n0|jdi|}n"d|vr |j|d<|jjdi||}|||d}|S)Nrr@generation_config) model_inputsrrr8)popr#rrTrrJgenerate)r.rrgenerate_kwargsrr model_outputss r_forwardz'TableQuestionAnsweringPipeline._forward{s  )) 9   ?3$3CClCC.$.>>>>#/997;7M 34)dj)LLLLOLLG)5RYZZ r!c|d}|d|d}jdkrejrw|dd\}}j|||}|\}}fdt |Djjjfdt |D} n/|d}j||}|d}ii} g} t |D]\} } fd | D} | d }| | d }|d | z| fd | Dd }|r||d<| |t|dkrtdn&dj |dD} t| dkr| n| dS)Nrrrr@rcFi|]\}}|jjj|Sr8)rJrKr>)rrpredr.s rrez>TableQuestionAnsweringPipeline.postprocess..s/wwwQXQRTXq$*"3"Ft"Lwwwr!c:i|]\}}|k ||dzS)z > r8)rrr aggregatorsno_agg_label_indexs rrez>TableQuestionAnsweringPipeline.postprocess..s;&&&29!T[_cu[u[uA{1~-[u[u[ur!rc*g|]}j|Sr8iatr coordinaters r z>TableQuestionAnsweringPipeline.postprocess..s MMM::.MMMr!rz, c*g|]}j|Sr8rrs rrz>TableQuestionAnsweringPipeline.postprocess..s RRR ei 3RRRr!)answer coordinatescells aggregatorz Empty answercg|]}d|iS)rr8)rrs rrz>TableQuestionAnsweringPipeline.postprocess..swwwf&)wwwr!T)skip_special_tokensr)r#rLrconvert_logits_to_predictionsrrJrKno_aggregation_label_indexr&joinrwr(r batch_decode)r.rrSrr[ logits_agg predictionsanswer_coordinates_batchagg_predictionsaggregators_prefixanswersrrrraggregator_prefixrrrrs` @@@r postprocessz*TableQuestionAnsweringPipeline.postprocesssK~.g& * 9  ~ (%,RaR[" "nJJ6SY[eff ( %'"G&/0H&I&I ' '"{MMMMMMM(__UB77 $6$:$:5"$E$E!/$))E2B2BB#.RRRRkRRR 6+5F<(v&&&&6{{a'777 xw8S8ST[qu8S8v8vwwwGg,,**ww :r!)NNN)NTN)F)r4r5r6r7default_input_namesrrBrTrr3rrrr __classcell__)rQs@rr;r;Us<(#H#H#J#JZZZZZZ"$$$@t@t@tDPPPPPd 5 5 5 5 "(;(;(;(;(;(;(;r!r;)r}r+rpr_utilsrrrrbaser r r r r rsmodels.auto.modeling_autorr tensorflowr=models.auto.modeling_tf_autorrrr;r8r!rrs  baaaaaaaaaaaaaLLL ? 2#2#2#2#2#O2#2#2#j,,4@@@AA^;^;^;^;^;X^;^;BA^;^;^;r!