
    g                        d dl Z d dlZd dlmZ d dlmZmZ d dlZd dl	m	Z	 ddl
mZ ddlmZmZmZ ddlmZmZmZ d	d
lmZ h dZ e            r
d dlZd dlmZ  e            rd dlZ ej        e          Zd Zd Zd Z d Z!d Z"defdZ#	 	 	 	 d$dZ$ G d de          Z% G d de%          Z& G d de%          Z' G d d          Z( G d  d!          Z) G d" d#          Z*dS )%    N)partial)Pool	cpu_count)tqdm   )whitespace_tokenize)BatchEncodingPreTrainedTokenizerBaseTruncationStrategy)is_tf_availableis_torch_availablelogging   )DataProcessor>   bartmpnetroberta	camembert)TensorDatasetc                    d                     |                    |                    }t          ||dz             D ]G}t          ||dz
  d          D ]0}d                     | ||dz                      }||k    r||fc c S 1H||fS )zFReturns tokenized answer spans that better match the annotated answer. r   )jointokenizerange)	
doc_tokensinput_start	input_end	tokenizerorig_answer_texttok_answer_text	new_startnew_end	text_spans	            ^/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/data/processors/squad.py_improve_answer_spanr&   +   s    hhy112BCCDDO;	A66 , ,	Y	Ar:: 	, 	,GI1,E!FGGIO++!7++++++ ,	,
 ##    c                     d}d}t          |           D ]`\  }}|j        |j        z   dz
  }||j        k     r#||k    r*||j        z
  }||z
  }	t          ||	          d|j        z  z   }
||
|k    r|
}|}a||k    S ):Check if this is the 'max context' doc span for the token.Nr   {Gz?)	enumeratestartlengthmin	doc_spanscur_span_indexposition
best_scorebest_span_index
span_indexdoc_spanendnum_left_contextnum_right_contextscores              r%   _check_is_max_contextr;   8   s    JO )) 4 4 ) )
Hnx.2hn$$c>>#hn4(N$&7884(/;QQ!3!3J(O_,,r'   c                     d}d}t          |           D ]e\  }}|d         |d         z   dz
  }||d         k     r&||k    r-||d         z
  }||z
  }	t          ||	          d|d         z  z   }
||
|k    r|
}|}f||k    S )r)   Nr,   r-   r   r*   )r+   r.   r/   s              r%   _new_check_is_max_contextr=   L   s     JO )) 4 4 ) )
Hw(8"44q8hw'''c>>#hw&77(N$&7884(8BT;TT!3!3J(O_,,r'   c                 `    | dk    s%| dk    s| dk    s| dk    st          |           dk    rdS dS )Nr   	
i/   TF)ord)cs    r%   _is_whitespacerD   b   s=    Cxx199T		Q$YY#a&&F:J:Jt5r'   c                    g }|r| j         s| j        }| j        }d                    | j        ||dz                      }	d                    t          | j                            }
|	                    |
          dk    r#t          	                    d|	 d|
 d           g S g }g }g }t          | j                  D ]\  }}|                    t          |                     t          j        j        dv rt                              |d	          }nt                              |          }|D ],}|                    |           |                    |           -|rz| j         ss|| j                 }| j        t          | j                  dz
  k     r|| j        dz            dz
  }nt          |          dz
  }t#          |||t          | j                  \  }}g }t                              | j        d
d|          }t)          t                    j                            dd                                          }|t.          v rt          j        t          j        z
  dz   nt          j        t          j        z
  }t          j        t          j        z
  }|}t          |          |z  t          |          k     rt          j        dk    r|}|}t8          j        j        }n|}|}t8          j        j        }t                               |||||d||z
  t          |          z
  |z
  d          }tC          t          |          t          |          |z  z
  |t          |          z
  |z
            }t          j"        |d         v rt          j        dk    r4|d         d |d         #                    t          j"                           }nct          |d                   dz
  |d         d d d         #                    t          j"                  z
  } |d         | dz   d          }n|d         }t          $                    |          }!i }"tK          |          D ]G}t          j        dk    rt          |          |z   |z   n|}#|t          |          |z  |z            |"|#<   H||d<   |!|d<   |"|d<   t          |          |z   |d<   i |d<   t          |          |z  |d<   ||d<   |                    |           d|vsd|v rt          |d                   dk    rn,|d         }t          |          |z  t          |          k     tK          t          |                    D ]k}$tK          ||$         d                   D ]M}%tM          ||$|$|z  |%z             }&t          j        dk    r|%n||$         d         |%z   }#|&||$         d         |#<   Nl|D ]}'|'d         #                    t          j'                  }(tQ          j)        |'d                   })t          j        dk    rd|)t          |          |z   d <   n,d|)t          |'d                    t          |          |z    <   tQ          j*        |'d         t          j"        k              }*tQ          j+        t          ,                    |'d         d                    -                                }+d|)|*<   d|)|+<   d|)|(<   | j         },d}d}|rl|,sj|'d         }-|'d         |'d         z   dz
  }.d
}/||-k    r||.k    sd}/|/r|(}|(}d},n5t          j        dk    rd}0nt          |          |z   }0||-z
  |0z   }||-z
  |0z   }|                    t]          |'d         |'d         |'d         |(|)/                                dd|'d         |'d         |'d         |'d         |||,| j0                              |S )Nr   r   r   zCould not find answer: 'z' vs. '')RobertaTokenizerLongformerTokenizerBartTokenizerRobertaTokenizerFastLongformerTokenizerFastBartTokenizerFastT)add_prefix_spaceF)add_special_tokens
truncation
max_length	Tokenizer right)rO   paddingrP   return_overflowing_tokensstridereturn_token_type_ids	input_idsparagraph_lentokenstoken_to_orig_map*truncated_query_with_special_tokens_lengthtoken_is_max_contextr,   r-   overflowing_tokensr   lefttoken_type_ids)already_has_special_tokensattention_mask)
example_index	unique_idrY   r]   rZ   r[   start_positionend_positionis_impossibleqas_id)1rg   re   rf   r   r   r   answer_textfindloggerwarningr+   appendlenr   	__class____name__r   r&   encodequestion_texttypereplacelowerMULTI_SEP_TOKENS_TOKENIZERS_SETmodel_max_lengthmax_len_single_sentencemax_len_sentences_pairpadding_sider   ONLY_SECONDvalue
ONLY_FIRSTencode_plusr.   pad_token_idindexconvert_ids_to_tokensr   r=   cls_token_idnp	ones_likewhereasarrayget_special_tokens_masknonzeroSquadFeaturestolistrh   )1examplemax_seq_length
doc_stridemax_query_lengthpadding_strategyis_trainingfeaturesre   rf   actual_textcleaned_answer_texttok_to_orig_indexorig_to_tok_indexall_doc_tokensitoken
sub_tokens	sub_tokentok_start_positiontok_end_positionspanstruncated_querytokenizer_typesequence_added_tokenssequence_pair_added_tokensspan_doc_tokenstextspairsrO   encoded_dictrY   non_padded_idslast_padding_id_positionrZ   r[   r   doc_span_indexjis_max_contextspan	cls_indexp_maskpad_token_indicesspecial_token_indicesspan_is_impossible	doc_startdoc_endout_of_span
doc_offsets1                                                    r%   !squad_convert_example_to_featuresr   h   s    H 
70 
 /+ hhw1.LSTDT2UVWW!hh':7;N'O'OPP/00B66NN`k``J]```aaaINg011 - -5  ^!4!4555' ,
 
 
 #++ED+IIJJ"++E22J# 	- 	-I$$Q'''!!),,,,	-  	
70 	
.w/EF#g&8"9"9A"===01E1IJQN">22Q61E.0@)WM`2
 2
.	- E&&%DUe '  O )__-55k2FFLLNNN <<< 	"Y%FFJJ')*KK 
 "+!;i>^!^$O
e**z
!C$7$7
7
7!W,,#E#E+7=JJ#E#E+6<J ,,!$%&*!J._1E1EEHbb"& - 	
 	
 #e**z"99S1114NN
 

 !\+%>>>%00!-k!:;t\+=V=\=\]f]s=t=t;t!u [122Q6k9RSWSWUWSW9X9^9^_h_u9v9vv ) ".k!:;SVW;W;Y;Y!Z *+6N00@@}%% 	V 	VAHQH^biHiHiC((+@@1DDopE'8Uj9PST9T'Ue$$(5_%!'X,=()EHEYEY\qEqAB/1+, #E

Z 7W!.X\"""|33 L00SFZ9[5\5\`a5a5a&';<y e**z
!C$7$7
7
7|  E

++ R Ru^,_=>> 	R 	RA6unn_iNilmNmnnN )V33 >*+WX[\\ 
 DRE.!"89%@@	R  C
 C
%++I,BCC	 d#3455!W,,EFF3''*??AABB]^FCX'''C,@,@CX,X*YYZHT+%6):P%PQQ "
--d;.?\`-aa!
 !

')) 	 %& !()$% y$2 	I1 	I WI7md8n4q8GK&)338HG8S8S" I!*(%)"")V33!"JJ!$_!5!58M!MJ!3i!?*!L/);jH[!%&%&"?3%)*@%AH~"&':";-)0~  	
 	
 	
 	
& Or'   tokenizer_for_convertc                 
    | a d S N)r   )r   s    r%   &squad_convert_example_to_features_initr   7  s    %IIIr'   rP   FTc
           
      <   g t          |t                                }t          |t          |f          5 }
t	          t
          |||||          }t          t          |
                    || d          t          |           d|	                     ddd           n# 1 swxY w Y   g }d}d	}t          t                    d
|	           D ]7}|s|D ]*}||_
        ||_        |                    |           |dz  }+|dz  }8|~|dk    rt                      st          d          t          j        d D             t          j                  }t          j        d D             t          j                  }t          j        d D             t          j                  }t          j        d D             t          j                  }t          j        d D             t          j                  }t          j        d D             t          j                  }|sHt          j        |                    d	          t          j                  }t+          ||||||          }njt          j        d D             t          j                  }t          j        d D             t          j                  }t+          ||||||||          }|fS |dk    rt-                      st          d          fd}d|j        v r>t0          j        t0          j        t0          j        t0          j        t0          j        dt0          j        t0          j        t0          j        t0          j        t0          j        df}t1          j        dg          t1          j        dg          t1          j        dg          t1          j        g           t1          j        g           dt1          j        g           t1          j        g           t1          j        g           t1          j        dg          t1          j        g           df}nt0          j        t0          j        t0          j        t0          j        dt0          j        t0          j        t0          j        t0          j        t0          j        df}t1          j        dg          t1          j        dg          t1          j        g           t1          j        g           dt1          j        g           t1          j        g           t1          j        g           t1          j        dg          t1          j        g           df}t0          j        j                            |||          S S )a  
    Converts a list of examples into a list of features that can be directly given as input to a model. It is
    model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs.

    Args:
        examples: list of [`~data.processors.squad.SquadExample`]
        tokenizer: an instance of a child of [`PreTrainedTokenizer`]
        max_seq_length: The maximum sequence length of the inputs.
        doc_stride: The stride used when the context is too large and is split across several features.
        max_query_length: The maximum length of the query.
        is_training: whether to create features for model evaluation or model training.
        padding_strategy: Default to "max_length". Which padding strategy to use
        return_dataset: Default False. Either 'pt' or 'tf'.
            if 'pt': returns a torch.data.TensorDataset, if 'tf': returns a tf.data.Dataset
        threads: multiple processing threads.


    Returns:
        list of [`~data.processors.squad.SquadFeatures`]

    Example:

    ```python
    processor = SquadV2Processor()
    examples = processor.get_dev_examples(data_dir)

    features = squad_convert_examples_to_features(
        examples=examples,
        tokenizer=tokenizer,
        max_seq_length=args.max_seq_length,
        doc_stride=args.doc_stride,
        max_query_length=args.max_query_length,
        is_training=not evaluate,
    )
    ```)initializerinitargs)r   r   r   r   r       )	chunksizez"convert squad examples to features)totaldescdisableNi ʚ;r   zadd example index and unique idr   ptz6PyTorch must be installed to return a PyTorch dataset.c                     g | ]	}|j         
S  )rX   .0fs     r%   
<listcomp>z6squad_convert_examples_to_features.<locals>.<listcomp>      %D%D%Daak%D%D%Dr'   )dtypec                     g | ]	}|j         
S r   )rb   r   s     r%   r   z6squad_convert_examples_to_features.<locals>.<listcomp>  s    +O+O+OA,<+O+O+Or'   c                     g | ]	}|j         
S r   )r`   r   s     r%   r   z6squad_convert_examples_to_features.<locals>.<listcomp>  s    *N*N*N1+;*N*N*Nr'   c                     g | ]	}|j         
S r   )r   r   s     r%   r   z6squad_convert_examples_to_features.<locals>.<listcomp>  r   r'   c                     g | ]	}|j         
S r   )r   r   s     r%   r   z6squad_convert_examples_to_features.<locals>.<listcomp>  s    ">">">18">">">r'   c                     g | ]	}|j         
S r   )rg   r   s     r%   r   z6squad_convert_examples_to_features.<locals>.<listcomp>  s    )L)L)La!/)L)L)Lr'   c                     g | ]	}|j         
S r   )re   r   s     r%   r   z6squad_convert_examples_to_features.<locals>.<listcomp>  s    /S/S/SQ0@/S/S/Sr'   c                     g | ]	}|j         
S r   )rf   r   s     r%   r   z6squad_convert_examples_to_features.<locals>.<listcomp>  s    -O-O-Oan-O-O-Or'   tfz<TensorFlow must be installed to return a TensorFlow dataset.c               3   6  K   t                    D ]\  } }|j        :|j        |j        | |j        d|j        |j        |j        |j        |j	        dfV  F|j        |j        |j        | |j        d|j        |j        |j        |j        |j	        dfV  d S )NrX   rb   feature_indexrh   start_positionsend_positionsr   r   rg   rX   rb   r`   r   rh   )
r+   r`   rX   rb   rh   re   rf   r   r   rg   )r   exr   s     r%   genz/squad_convert_examples_to_features.<locals>.gen  s      "8,, ! !2$, *,.0.?-.&(i	  02/@-/_)+&(i-/-=     $ *,.0.?.0.?-.&(i  02/@-/_)+&(i-/-=     %! !r'   r`   r   r   r   ) r.   r   r   r   r   r   listr   imaprn   rc   rd   rm   r   RuntimeErrortorchtensorlongfloatarangesizer   r   model_input_namesr   int32int64stringTensorShapedataDatasetfrom_generator)examplesr   r   r   r   r   r   return_datasetthreadstqdm_enabledp	annotate_new_featuresrd   rc   example_featuresexample_featureall_input_idsall_attention_masksall_token_type_idsall_cls_index
all_p_maskall_is_impossibleall_feature_indexdatasetall_start_positionsall_end_positionsr   train_typestrain_shapesr   s                                 @r%   "squad_convert_examples_to_featuresr  <  s   ` H'9;;''G	g#IU^T`	a	a	a 
ef-)!--#
 
 
	 y(b99(mm9((	  
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
$ LIM H,M[gWg   
 
   	/ 	 	O,9O)(1O%000NIIH!## 	YWXXX %D%D8%D%D%DEJWWW#l+O+Oh+O+O+OW\Wabbb"\*N*NX*N*N*NV[V`aaa%D%D8%D%D%DEJWWW\">">X">">">ekRRR
!L)L)L8)L)L)LTYT_``` 	 %]-?-?-B-B%* U U U#24FHY[hjt GG #(,/S/S(/S/S/S[`[e"f"f"f %-O-Oh-O-O-OW\Wa b b b##"#!!	 	G   	4		   	_]^^^"	 "	 "	 "	 "	J y::: "$&(h&(h%'X i  (*x%'X!# h%'X K& "$!7!7&(ndV&<&<&(ndV&<&<%'^B%7%7 nR00  (*~b'9'9%'^B%7%7!#!3!3 ndV44%'^B%7%7 LL$ !h"(UWU]ikirss')x%'X!# h%'X 	K "$!7!7&(ndV&<&<%'^B%7%7 nR00	  (*~b'9'9%'^B%7%7!#!3!3 ndV44%'^B%7%7 L  w--c;MMMs   AB##B'*B'c                   @    e Zd ZdZdZdZd	dZd	dZd
dZd
dZ	d Z
dS )SquadProcessorz
    Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
    version 2.0 of SQuAD, respectively.
    NFc           
         |sb|d         d         d                                                              d          }|d         d         d                                          }g }n6d t          |d         d         |d         d                   D             }d }d }t          |d                                                              d          |d                                                              d          |d	                                                              d          |||d
                                                              d          |          S )Nanswerstextr   utf-8answer_startc                     g | ]A\  }}|                                 |                                                     d           dBS )r  )r	  r  )numpydecode)r   r,   r  s      r%   r   z@SquadProcessor._get_example_from_tensor_dict.<locals>.<listcomp>,  sR       E4 "'

8K8KG8T8TUU  r'   idquestioncontexttitle)rh   rr   context_textri   start_position_characterr  r  )r  r  zipSquadExample)selftensor_dictevaluateanswerr	  r  s         r%   _get_example_from_tensor_dictz,SquadProcessor._get_example_from_tensor_dict&  sQ    	  +F3A6<<>>EEgNNF&y1.A!DJJLLLGG #&{9'=n'M{[dOeflOm#n#n  G
 FLt$**,,33G<<%j17799@@II$Y/5577>>wGG%1g&,,..55g>>
 
 
 	
r'   c                     |r	|d         }n|d         }g }t          |          D ],}|                    |                     ||                     -|S )av  
        Creates a list of [`~data.processors.squad.SquadExample`] using a TFDS dataset.

        Args:
            dataset: The tfds dataset loaded from *tensorflow_datasets.load("squad")*
            evaluate: Boolean specifying if in evaluation mode or in training mode

        Returns:
            List of SquadExample

        Examples:

        ```python
        >>> import tensorflow_datasets as tfds

        >>> dataset = tfds.load("squad")

        >>> training_examples = get_examples_from_dataset(dataset, evaluate=False)
        >>> evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)
        ```
validationtrain)r  )r   rm   r  )r  r   r  r   r  s        r%   get_examples_from_datasetz(SquadProcessor.get_examples_from_dataset>  sl    ,  	'l+GGg&G== 	` 	`KOOD>>{U]>^^____r'   c                 8   |d}| j         t          d          t          t          j                            ||| j         n|          dd          5 }t          j        |          d         }ddd           n# 1 swxY w Y   |                     |d          S )	a  
        Returns the training examples from the data directory.

        Args:
            data_dir: Directory containing the data files used for training and evaluating.
            filename: None by default, specify this if the training file has a different name than the original one
                which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively.

        NrR   NSquadProcessor should be instantiated via SquadV1Processor or SquadV2Processorrr  encodingr   r  )	
train_file
ValueErroropenospathr   jsonload_create_examplesr  data_dirfilenamereader
input_datas        r%   get_train_examplesz!SquadProcessor.get_train_examples_  s     H?"mnnnGLLh6F4??HUUWZel
 
 
 	36**62J	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 $$Z999   A;;A?A?c                 8   |d}| j         t          d          t          t          j                            ||| j         n|          dd          5 }t          j        |          d         }ddd           n# 1 swxY w Y   |                     |d          S )	a  
        Returns the evaluation example from the data directory.

        Args:
            data_dir: Directory containing the data files used for training and evaluating.
            filename: None by default, specify this if the evaluation file has a different name than the original one
                which is `dev-v1.1.json` and `dev-v2.0.json` for squad versions 1.1 and 2.0 respectively.
        NrR   r  r   r  r!  r   dev)	dev_filer$  r%  r&  r'  r   r(  r)  r*  r+  s        r%   get_dev_exampleszSquadProcessor.get_dev_examplesu  s     H= mnnnGLLH4D4==(SSUXcj
 
 
 	36**62J	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 $$Z777r1  c                    |dk    }g }t          |          D ]}|d         }|d         D ]}|d         }|d         D ]}	|	d         }
|	d         }d }d }g }|	                    dd	          }|s)|r|	d
         d         }|d         }|d         }n|	d
         }t          |
|||||||          }|                    |           |S )Nr  r  
paragraphsr  qasr  r  rg   Fr  r   r  r	  )rh   rr   r  ri   r  r  rg   r  )r   getr  rm   )r  r/  set_typer   r   entryr  	paragraphr  qarh   rr   r  ri   r  rg   r  r   s                     r%   r*  zSquadProcessor._create_examples  s*   ')*%% 	- 	-E'NE"<0 - -	(3#E* - -BXF$&zNM/3,"&K G$&FF?E$B$BM( 4& 4%'	]1%5F*0.K7=n7M44&(mG*%&3%1$/1I#&3 '	 	 	G OOG,,,,5--: r'   )Fr   )rp   
__module____qualname____doc__r#  r4  r  r  r0  r5  r*  r   r'   r%   r  r    s         
 JH
 
 
 
0   B: : : :,8 8 8 8*" " " " "r'   r  c                       e Zd ZdZdZdS )SquadV1Processorztrain-v1.1.jsonzdev-v1.1.jsonNrp   r>  r?  r#  r4  r   r'   r%   rB  rB            "JHHHr'   rB  c                       e Zd ZdZdZdS )SquadV2Processorztrain-v2.0.jsonzdev-v2.0.jsonNrC  r   r'   r%   rF  rF    rD  r'   rF  c                       e Zd ZdZg dfdZdS )r  aT  
    A single training/test example for the Squad dataset, as loaded from disk.

    Args:
        qas_id: The example's unique identifier
        question_text: The question string
        context_text: The context string
        answer_text: The answer string
        start_position_character: The character position of the start of the answer
        title: The title of the example
        answers: None by default, this is used during evaluation. Holds answers as well as their start positions.
        is_impossible: False by default, set to True if the example has no possible answer.
    Fc	                 *   || _         || _        || _        || _        || _        || _        || _        d\  | _        | _        g }	g }
d}| j        D ]c}t          |          rd}n*|r|	
                    |           n|	dxx         |z  cc<   d}|

                    t          |	          dz
             d|	| _        |
| _        |O|sO|
|         | _        |
t          |t          |          z   dz
  t          |
          dz
                     | _        d S d S d S )N)r   r   Tr   Fr   )rh   rr   r  ri   r  rg   r  re   rf   rD   rm   rn   r   char_to_word_offsetr.   )r  rh   rr   r  ri   r  r  r  rg   r   rI  prev_is_whitespacerC   s                r%   __init__zSquadExample.__init__  s^    *(&
*15.T.
 ! " 		< 		<Aa   +%)""% (%%a((((rNNNa'NNN%*"&&s:':;;;;$#6  $//"56N"OD 3,s;/?/??!CSI\E]E]`aEabb!D 0///r'   Nrp   r>  r?  r@  rK  r   r'   r%   r  r    s:         , - - - - - -r'   r  c                   (    e Zd ZdZ	 	 ddedefdZdS )r   a  
    Single squad example features to be fed to a model. Those features are model-specific and can be crafted from
    [`~data.processors.squad.SquadExample`] using the
    :method:*~transformers.data.processors.squad.squad_convert_examples_to_features* method.

    Args:
        input_ids: Indices of input sequence tokens in the vocabulary.
        attention_mask: Mask to avoid performing attention on padding token indices.
        token_type_ids: Segment token indices to indicate first and second portions of the inputs.
        cls_index: the index of the CLS token.
        p_mask: Mask identifying tokens that can be answers vs. tokens that cannot.
            Mask with 1 for tokens than cannot be in the answer and 0 for token that can be in an answer
        example_index: the index of the example
        unique_id: The unique Feature identifier
        paragraph_len: The length of the context
        token_is_max_context:
            List of booleans identifying which tokens have their maximum context in this feature object. If a token
            does not have their maximum context in this feature object, it means that another feature object has more
            information related to that token and should be prioritized over this feature for that token.
        tokens: list of tokens corresponding to the input ids
        token_to_orig_map: mapping between the tokens and the original text, needed in order to identify the answer.
        start_position: start of the answer token index
        end_position: end of the answer token index
        encoding: optionally store the BatchEncoding with the fast-tokenizer alignment methods.
    Nrh   r"  c                     || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        || _
        || _        || _        || _        || _        || _        d S r   )rX   rb   r`   r   r   rc   rd   rY   r]   rZ   r[   re   rf   rg   rh   r"  )r  rX   rb   r`   r   r   rc   rd   rY   r]   rZ   r[   re   rf   rg   rh   r"  s                    r%   rK  zSquadFeatures.__init__  s    & #,,"*"*$8!!2,(* r'   )NN)rp   r>  r?  r@  strr	   rK  r   r'   r%   r   r     sT         T "&#%! %!  !%!"  #%! %! %! %! %! %!r'   r   c                       e Zd ZdZddZdS )SquadResultaJ  
    Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset.

    Args:
        unique_id: The unique identifier corresponding to that example.
        start_logits: The logits corresponding to the start of the answer
        end_logits: The logits corresponding to the end of the answer
    Nc                 b    || _         || _        || _        |r|| _        || _        || _        d S d S r   )start_logits
end_logitsrd   start_top_indexend_top_index
cls_logits)r  rd   rS  rT  rU  rV  rW  s          r%   rK  zSquadResult.__init__E  sE    ($" 	)#2D !.D(DOOO	) 	)r'   )NNNrL  r   r'   r%   rQ  rQ  ;  s2         ) ) ) ) ) )r'   rQ  )rP   Fr   T)+r(  r&  	functoolsr   multiprocessingr   r   r  r   r   models.bert.tokenization_bertr   tokenization_utils_baser	   r
   r   utilsr   r   r   r   rv   r   torch.utils.datar   
tensorflowr   
get_loggerrp   rk   r&   r;   r=   rD   r   r   r  r  rB  rF  r  r   rQ  r   r'   r%   <module>r`     s    				       + + + + + + + +           @ @ @ @ @ @ a a a a a a a a a a A A A A A A A A A A             #L"K"K   /LLL......? 		H	%	%
$ 
$ 
$- - -(- - -,  L L L^&BY & & & & "^ ^ ^ ^BO O O O O] O O Od    ~   
    ~   
< < < < < < < <~@! @! @! @! @! @! @! @!F) ) ) ) ) ) ) ) ) )r'   