g6ddlZddlZddlZddlmZddlmZmZmZddlm Z m Z m Z e j e ZeGddZed Gd d ZGd d ZGddeZdS)N) dataclass)ListOptionalUnion)is_tf_availableis_torch_availableloggingcbeZdZUdZeed<eed<dZeeed<dZeeed<dZ dS) InputExamplea5 A single training/test example for simple sequence classification. Args: guid: Unique id for the example. text_a: string. The untokenized text of the first sequence. For single sequence tasks, only this sequence must be specified. text_b: (Optional) string. The untokenized text of the second sequence. Only must be specified for sequence pair tasks. label: (Optional) string. The label of the example. This should be specified for train and dev examples, but not for test examples. guidtext_aNtext_blabelcXtjtj|ddzS)*Serializes this instance to a JSON string.)indent jsondumps dataclassesasdictselfs ^/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/data/processors/utils.pyto_json_stringzInputExample.to_json_string1s'z+,T221===DD) __name__ __module__ __qualname____doc__str__annotations__rrrrrrr r sp   III KKK FHSM   E8C=EEEEErr T)frozenceZdZUdZeeed<dZeeeed<dZ eeeed<dZ ee ee fed<dZ dS) InputFeaturesa A single set of features of data. Property names are the same names as the corresponding inputs to a model. Args: input_ids: Indices of input sequence tokens in the vocabulary. attention_mask: Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: Usually `1` for tokens that are NOT MASKED, `0` for MASKED (padded) tokens. token_type_ids: (Optional) Segment token indices to indicate first and second portions of the inputs. Only some models use them. label: (Optional) Label corresponding to the input. Int for classification problems, float for regression problems. input_idsNattention_masktoken_type_idsrcTtjtj|dzS)rrrrs rrzInputFeatures.to_json_stringKs"z+,T2233d::r)r r!r"r#rintr%r+rr,rrfloatrr&rrr)r)6s  Cy*.NHT#Y'...*.NHT#Y'...)-E8E#u*% &---;;;;;rr)cNeZdZdZdZdZdZdZdZdZ e d d Z dS) DataProcessorzEBase class for data converters for sequence classification data sets.ct)z Gets an example from a dict with tensorflow tensors. Args: tensor_dict: Keys and values should match the corresponding Glue tensorflow_dataset examples. NotImplementedError)r tensor_dicts rget_example_from_tensor_dictz*DataProcessor.get_example_from_tensor_dictSs"###rct)z8Gets a collection of [`InputExample`] for the train set.r3rdata_dirs rget_train_examplesz DataProcessor.get_train_examples]!###rct)z6Gets a collection of [`InputExample`] for the dev set.r3r8s rget_dev_exampleszDataProcessor.get_dev_examplesar;rct)z7Gets a collection of [`InputExample`] for the test set.r3r8s rget_test_exampleszDataProcessor.get_test_exampleser;rct)z*Gets the list of labels for this data set.r3rs r get_labelszDataProcessor.get_labelsir;rct|dkr1|t|j|_|S)z Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts examples to the correct format. )lenrAr.r)rexamples rtfds_mapzDataProcessor.tfds_mapmsF t  ! !A % % OO--c'-.@.@AGMrNct|dd5}ttj|d|cdddS#1swxYwYdS)z!Reads a tab separated value file.rz utf-8-sig)encoding ) delimiter quotecharN)openlistcsvreader)cls input_filerLfs r _read_tsvzDataProcessor._read_tsvvs*cK 8 8 8 LA 1 JJJKK L L L L L L L L L L L L L L L L L Ls$AA AN) r r!r"r#r6r:r=r?rArF classmethodrTr&rrr1r1PsOO$$$$$$$$$$$$$$$LLL[LLLrr1ceZdZdZddZdZdZe dd Zedd Z dd Z ddZ ddZ dS)%SingleSentenceClassificationProcessorz@Generic processor for a single sentence classification data set.NclassificationFcN|gn||_|gn||_||_||_dSrU)labelsexamplesmodeverbose)rr[r\r]r^s r__init__z.SingleSentenceClassificationProcessor.__init__s3"Nbb &.H   rc*t|jSrU)rDr\rs r__len__z-SingleSentenceClassificationProcessor.__len__s4=!!!rct|tr!t|j|j|S|j|S)N)r[r\) isinstanceslicerXr[r\)ridxs r __getitem__z1SingleSentenceClassificationProcessor.__getitem__sA c5 ! ! j8 VZVcdgVhiii i}S!!rrrCc P|di|}|||||||dd|S)NT) split_name column_label column_text column_idskip_first_rowoverwrite_labelsoverwrite_examplesr&)add_examples_from_csv) rQ file_namerirjrkrlrmkwargs processors rcreate_from_csvz5SingleSentenceClassificationProcessor.create_from_csvsRCMM&MM '' !%#)!# ( rc D|di|}||||S)N)r[r&) add_examples)rQtexts_or_text_and_labelsr[rrrss rcreate_from_examplesz:SingleSentenceClassificationProcessor.create_from_exampless3CMM&MM 7GGGrc ||} |r | dd} g} g} g} t| D]\} }| ||| |||| ||Y|r|d| nt| }| ||| | | ||S)NrC-)rnro)rT enumerateappendr$rv)rrqrirjrkrlrmrnrolinestextsr[idsiliner s rrpz;SingleSentenceClassificationProcessor.add_examples_from_csvsy))  !""IE '' ! !GAt LLk* + + + MM$|, - - -$ 4 ?++++.8D***q***c!ff 4      631AVh!   rc |Ot|t|kr/tdt|dt||Ot|t|kr/tdt|dt||dgt|z}|dgt|z}g}t}t|||D]g\}} } t |t t fr| |\} } n|} || |t| | d| h|r||_ n|j ||rt ||_ n9t t|j  ||_ |j S)Nz(Text and labels have mismatched lengths z and z%Text and ids have mismatched lengths )r rrr)rD ValueErrorsetziprctuplerNaddr|r r\extendr[union) rrwr[rrnror\ added_labelstext_or_text_and_labelrr texts rrvz2SingleSentenceClassificationProcessor.add_exampless  #&>"?"?3v;;"N"Nl3?W;X;Xll_bci_j_jll  ?s#;<<CHHsSIaEbEbssilmpiqiqsstt t ;&37888C >Vc":;;;Fuu 367OQWY\3]3] \ \ / "E405$-@@ .U]4 ee-   U # # # OOLd4TYZZZ [ [ [ [  +$DMM M  * * *  E|,,DKKs4;//55lCCDDDK}rTc  ||j}dt|jD}g}t|jD]p\} } | dzdkrtd| || jdt||j} | | qtd|D} gtt||jD][\} \} } | dzdkr2td | d t|j|rd ndgt| z} | t| z }|r|g|z| z} |rdnd g|z| z} n| |g|zz} | |rdnd g|zz} t| | kr"td t| d | t| | kr"td t| d | |jdkr|| j}n4|jdkrt!| j}nt|j| dkr|jrtdtd| jtddd| Dtddd| Dtd| jd|d t)| | |]|S|dkrt+st-dddl}fd}|jj||j|jd|jf|dg|dgd|gf}|S|d krt=st-d!ddl}dd"l m!}|"d#D|j#$}|"d%D|j#$}|jdkr'|"d&D|j#$}n1|jdkr&|"d'D|j$}||||}|Std())a Convert examples in a list of `InputFeatures` Args: tokenizer: Instance of a tokenizer that will tokenize the examples max_length: Maximum example length pad_on_left: If set to `True`, the examples will be padded on the left rather than on the right (default) pad_token: Padding token mask_padding_with_zero: If set to `True`, the attention mask will be filled by `1` for actual values and by `0` for padded values. If set to `False`, inverts it (`1` for padded values, `0` for actual values) Returns: If the `examples` input is a `tf.data.Dataset`, will return a `tf.data.Dataset` containing the task-specific features. If the input is a list of `InputExamples`, will return a list of task-specific `InputFeatures` which can be fed to the model. Nci|]\}}|| Sr&r&).0rrs r zFSingleSentenceClassificationProcessor.get_features..sEEE(!UUAEEEri'rzTokenizing example T)add_special_tokens max_lengthc34K|]}t|VdSrU)rD)rr*s r zESingleSentenceClassificationProcessor.get_features..s(IIi3y>>IIIIIIrzWriting example /rCzError with input length z vs rY regressionz*** Example ***zguid: z input_ids:  c,g|]}t|Sr&r$rxs r zFSingleSentenceClassificationProcessor.get_features..6s3N3N3NqCFF3N3N3Nrzattention_mask: c,g|]}t|Sr&rrs rrzFSingleSentenceClassificationProcessor.get_features..7s8X8X8XAQ8X8X8Xrzlabel: z (id = )r*r+rtfz?return_tensors set to 'tf' but TensorFlow 2.0 can't be importedc3FKD]}|j|jd|jfVdS)Nr*r+r)exfeaturess rgenz?SingleSentenceClassificationProcessor.get_features..genCsI"ggB)+IZ[[]_]efffffggrrptz8return_tensors set to 'pt' but PyTorch can't be imported) TensorDatasetcg|] }|j Sr&)r*rrSs rrzFSingleSentenceClassificationProcessor.get_features..Ss)H)H)H!!+)H)H)Hr)dtypecg|] }|j Sr&)r+rs rrzFSingleSentenceClassificationProcessor.get_features..Ts.R.R.RAq/?.R.R.Rrcg|] }|j Sr&rrs rrzFSingleSentenceClassificationProcessor.get_features..V*E*E*Eq17*E*E*Ercg|] }|j Sr&rrs rrzFSingleSentenceClassificationProcessor.get_features..Xrrz,return_tensors should be one of 'tf' or 'pt')$max_lenr{r[r\loggerinfoencoderminr|maxrrDrr]rr/r^r joinr)r RuntimeError tensorflowdataDatasetfrom_generatorint32int64 TensorShaper torchtorch.utils.datartensorlong)r tokenizerr pad_on_left pad_tokenmask_padding_with_zeroreturn_tensors label_map all_input_idsex_indexrEr* batch_lengthr+padding_lengthrrrdatasetrrall_attention_mask all_labelsrs @r get_featuresz2SingleSentenceClassificationProcessor.get_featuress6  "*JEEi .D.DEEE  !*4=!9!9 , , Hg%1$$ <(<<===!((#'z9+<==)I   + + + +II=IIIII .7M4=8Y8Y.Z.Z# l# l *H*y'%1$$ NxNN#dm:L:LNNOOO$:@aaqAC NNRN*C NN:N j'[>9YF (>#E11A"F"W[i!i%)~)EF !/9O4VAAUV3WZh3h!i9~~-- !^C NN!^!^P\!^!^___>""l22 !cCkp####P#uMuMuMuMuMuMrrX)rOrrrtypingrrrutilsrr r get_loggerr rr r)r1rXr&rrrs~"  !!!!!!((((((((((AAAAAAAAAA  H % % EEEEEEE E0 $;;;;;;;;2*L*L*L*L*L*L*L*LZ`M`M`M`M`MM`M`M`M`M`Mr