Ng4fddlmZddlZddlmZddlmZddlmZm Z ddl m Z ddl Z ddl mZmZmZddlmZerdd lmZejeZGd d ZGd d eeZGddeeZGddeeZGddeeZGddeeZdS)) annotationsN) defaultdict)Iterator) accumulatecycle)Any) BatchSampler ConcatDatasetSubsetRandomSampler)is_datasets_available)Datasetc,eZdZdZdfd Zd dZxZS) SetEpochMixinz Required for a BatchSampler as the Trainer will call set_epoch on the BatchSampler at the beginning of each epoch. The BatchSampler can then set the generator seed accordingly. returnNonecHtj|i|d|_dS)Nr)super__init__epoch)selfargskwargs __class__s Y/var/www/html/ai-engine/env/lib/python3.11/site-packages/sentence_transformers/sampler.pyrzSetEpochMixin.__init__s*$)&))) rintc||_dSN)r)rrs r set_epochzSetEpochMixin.set_epochs  r)rr)rrrr)__name__ __module__ __qualname____doc__rr __classcell__rs@rrrs[  rrceZdZdZdS)DefaultBatchSamplera This sampler is the default batch sampler used in the SentenceTransformer library. It is equivalent to the PyTorch BatchSampler. Args: sampler (Sampler or Iterable): The sampler used for sampling elements from the dataset, such as SubsetRandomSampler. batch_size (int): Number of samples per batch. drop_last (bool): If True, drop the last incomplete batch if the dataset size is not divisible by the batch size. N)r r!r"r#rrr'r'#s    rr'cLeZdZdZ ddfd ZeddZddZxZS)GroupByLabelBatchSampleraK This sampler groups samples by their labels and aims to create batches such that each batch contains samples where the labels are as homogeneous as possible. This sampler is meant to be used alongside the ``Batch...TripletLoss`` classes, which require that each batch contains at least 2 examples per label class. Recommended for: - :class:`~sentence_transformers.losses.BatchAllTripletLoss` - :class:`~sentence_transformers.losses.BatchHardSoftMarginTripletLoss` - :class:`~sentence_transformers.losses.BatchHardTripletLoss` - :class:`~sentence_transformers.losses.BatchSemiHardTripletLoss` Args: dataset (Dataset): The dataset to sample from. batch_size (int): Number of samples per batch. Must be divisible by 2. drop_last (bool): If True, drop the last incomplete batch if the dataset size is not divisible by the batch size. valid_label_columns (List[str]): List of column names to check for labels. The first column name from ``valid_label_columns`` found in the dataset will be used as the label column. generator (torch.Generator, optional): Optional random number generator for shuffling the indices. seed (int, optional): Seed for the random number generator to ensure reproducibility. Nrdatasetr batch_sizer drop_lastboolvalid_label_columns list[str] generatortorch.Generatorseedrrc t|||||_||_||_||_||_|jdzdkrtd|||}tt}t|D] \} } ||  | ! fd| D|_dS)NzEThe batch size for `GroupByLabelBatchSampler` must be divisible by 2.cZi|]'\}}t|dzdzx||d(S)r5Nlen).0labelsample_indices num_sampless r z5GroupByLabelBatchSampler.__init__..csU   %~">22a7!;;  >,;,/   r)rrr+r,r-r1r3 ValueError_determine_labels_to_userlist enumerateappenditemsgroups) rr+r,r-r/r1r3labelsrE sample_idxr;r=rs @rrz!GroupByLabelBatchSampler.__init__Ks *i888 $"" ?Q ! # #dee e..w8KLLT""!*6!2!2 - - J 5M  , , , ,    )/    r list[Any]cl|pgD]}||jvr ||cStd|d|jd)Nz None of the valid_label_columns z3 are in the dataset, which only has these columns: .) column_namesr?)r+r/ column_names rr@z1GroupByLabelBatchSampler._determine_labels_to_useisy.4" , ,Kg222{++++3 E/B E E-4-A E E E   rIterator[list[int]]c#:K|jr.|jr'|j|j|jzg}t |j}tjt|j|jD]|}||}|j|}| |t||j kr8|d|j V||j d}t||j k8}|j s|r|VdSdSdS)Nr1) r1r3 manual_seedrrArEkeystorchrandpermr9extendr,r-)r partial_batch unique_labels label_idxr;sampless r__iter__z!GroupByLabelBatchSampler.__iter__ss? > ?di ? N & &ty4:'= > > > T[--//00 DK(8(8DNSSS A AI!),Ek%(G   ) ) )m$$77#$5do$56666 -do.?.? @ m$$77~ -          r)NNrr+r r,rr-r.r/r0r1r2r3rrr)r+r r/r0rrHrrM) r r!r"r#r staticmethodr@rYr$r%s@rr*r*1s<*.%)       <   \         rr*c8eZdZgddfdfd ZddZddZxZS)NoDuplicatesBatchSamplerNrr+r r,rr-r.r/r0r1r2r3rrc t|||t|jt|zx}r||}||_||_||_||_||_ dS)as This sampler creates batches such that each batch contains samples where the values are unique, even across columns. This is useful when losses consider other samples in a batch to be in-batch negatives, and you want to ensure that the negatives are not duplicates of the anchor/positive sample. Recommended for: - :class:`~sentence_transformers.losses.MultipleNegativesRankingLoss` - :class:`~sentence_transformers.losses.CachedMultipleNegativesRankingLoss` - :class:`~sentence_transformers.losses.MultipleNegativesSymmetricRankingLoss` - :class:`~sentence_transformers.losses.CachedMultipleNegativesSymmetricRankingLoss` - :class:`~sentence_transformers.losses.MegaBatchMarginLoss` - :class:`~sentence_transformers.losses.GISTEmbedLoss` - :class:`~sentence_transformers.losses.CachedGISTEmbedLoss` Args: dataset (Dataset): The dataset to sample from. batch_size (int): Number of samples per batch. drop_last (bool): If True, drop the last incomplete batch if the dataset size is not divisible by the batch size. valid_label_columns (List[str]): List of column names to check for labels. The first column name from ``valid_label_columns`` found in the dataset will be used as the label column. generator (torch.Generator, optional): Optional random number generator for shuffling the indices. seed (int, optional): Seed for the random number generator to ensure reproducibility. N) rrsetrKremove_columnsr+r,r-r1r3) rr+r,r-r/r1r3 label_columnsrs rrz!NoDuplicatesBatchSampler.__init__sF *i888 455z4NoDuplicatesBatchSampler.__iter__..sI!!!"U<<(899!?B^>S>S>S>S>SrN)r1r3rPrr`rRrSr9r+tolistrDrCr,updater-)rremaining_indices batch_values batch_indicesindex sample_valuess rrYz!NoDuplicatesBatchSampler.__iter__sq > ?di ? N & &ty4:'= > > >s4 Pt|$$7 7 %%7!;O OrrZr[rrr r!r"rrYrtr$r%s@rr^r^s *,%)*******X"4"4"4"4HPPPPPPPPrr^c:eZdZdZ ddfd ZddZddZxZS)RoundRobinBatchSamplera Batch sampler that yields batches in a round-robin fashion from multiple batch samplers, until one is exhausted. With this sampler, it's unlikely that all samples from each dataset are used, but we do ensure that each dataset is sampled from equally. Args: dataset (ConcatDataset): A concatenation of multiple datasets. batch_samplers (List[BatchSampler]): A list of batch samplers, one for each dataset in the ConcatDataset. generator (torch.Generator, optional): A generator for reproducible sampling. Defaults to None. seed (int, optional): A seed for the generator. Defaults to None. Nr+r batch_samplerslist[BatchSampler]r1r2r3rrrct|jt|krtdt||dj|dj||_||_||_ ||_ dS)NzTThe number of batch samplers must match the number of datasets in the ConcatDataset.r) r9datasetsr?rrr,r-r+ryr1r3rr+ryr1r3rs rrzRoundRobinBatchSampler.__init__s w C$7$7 7 7stt t ."3">q@Q@[\\\ ," rrMc#K|jr.|jr'|j|j|jzd|jjD}dgt t|z}d|jD}ttt|D]@}|| fdt||DV/#t$rYdSwxYwdS)Nc,g|]}t|Sr(r8r:r+s r z3RoundRobinBatchSampler.__iter__..IIIs7||IIIrrc,g|]}t|Sr(iterr:samplers rrz3RoundRobinBatchSampler.__iter__..KKKG$w--KKKrcg|]}|zSr(r(r:idx sample_offsets rrz3RoundRobinBatchSampler.__iter__..XXXss]*XXXr)r1r3rPrr+r|rArryrranger9next StopIteration)rr=sample_offsetsry dataset_idxrs @rrYzRoundRobinBatchSampler.__iter__s > ?di ? N & &ty4:'= > > >II4<3HIII tJ{$;$;<<<KKt7JKKK s>':':!;!;<<  K*;7M XXXXd>+;V6W6WXXXXXXX        s4#C C'&C'chtd|jDt|jzS)Nc34K|]}t|VdSrr8rs r z1RoundRobinBatchSampler.__len__.. s(CCG3w<<CCCCCCr)minryr9rss rrtzRoundRobinBatchSampler.__len__ s2CCt/BCCCCCc$J]F^F^^^r)NN r+r ryrzr1r2r3rrrr[ru)r r!r"r#rrYrtr$r%s@rrxrxs   &*        ________rrxc0eZdZdfd Zdd ZddZxZS)ProportionalBatchSamplerr+r ryrzr1r2r3rrrct||dj|dj||_||_||_||_dS)a Batch sampler that samples from each dataset in proportion to its size, until all are exhausted simultaneously. With this sampler, all samples from each dataset are used and larger datasets are sampled from more frequently. Args: dataset (ConcatDataset): A concatenation of multiple datasets. batch_samplers (List[BatchSampler]): A list of batch samplers, one for each dataset in the ConcatDataset. generator (torch.Generator, optional): A generator for reproducible sampling. Defaults to None. seed (int, optional): A seed for the generator. Defaults to None. rN)rrr,r-r+ryr1r3r}s rrz!ProportionalBatchSampler.__init__sR" ."3">q@Q@[\\\ ," rrMc#K|j|j|jzd|jjD}dgt t|z}d|jD}dt|D}t||j}d|jD}|D]>}|| fdt||DV/#t$rY;wxYwdS)Nc,g|]}t|Sr(r8rs rrz5ProportionalBatchSampler.__iter__..(rrrc,g|]}t|Sr(r8rs rrz5ProportionalBatchSampler.__iter__..+sGGGs7||GGGrc<g|]\}}t|D]}|Sr()r)r:rlength_s rrz5ProportionalBatchSampler.__iter__..,s3```;3RWX^R_R_``Q3````rrOc,g|]}t|Sr(rrs rrz5ProportionalBatchSampler.__iter__../rrcg|]}|zSr(r(rs rrz5ProportionalBatchSampler.__iter__..3rr) r1rPr3rr+r|rArryrBr rr) rr=r num_batchesdataset_indicesdataset_idx_samplerryrrs @rrYz!ProportionalBatchSampler.__iter__%s. ""49tz#9:::II4<3HIII tJ{$;$;<<<GG43FGGG ``)K2H2H```1/T^\\\KKt7JKKK.  K*;7M XXXXd>+;V6W6WXXXXXXX        s?#C## C0/C0c>td|jDS)Nc,g|]}t|Sr(r8rs rrz4ProportionalBatchSampler.__len__..8sDDDWCLLDDDr)sumryrss rrtz ProportionalBatchSampler.__len__7s"DD0CDDDEEErrr[rurvr%s@rrr sm.$FFFFFFFFrr) __future__rlogging collectionsrcollections.abcr itertoolsrrtypingrrRtorch.utils.datar r r sentence_transformers.utilr r|r getLoggerr loggerrr'r*r^rxrr(rrrs""""""######$$$$$$'''''''' MMMMMMMMMM<<<<<<!       8 $ $             -   Q Q Q Q Q }lQ Q Q hUPUPUPUPUP}lUPUPUPp-_-_-_-_-_]L-_-_-_`+F+F+F+F+F}l+F+F+F+F+Fr