
    NgM                        d dl mZ d dlmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlZd dlmZmZ d dlmZmZ d d	lmZ d d
lmZmZ  G d d          ZddZ G d dej                  ZdS )    )annotations)IterableIterator)nullcontext)partial)AnyN)Tensornn)get_device_statesset_device_states)SentenceTransformer)StaticEmbeddingTransformerc                  *    e Zd ZdZddZddZddZdS )	RandContexta  
    Random-state context manager class. Reference: https://github.com/luyug/GradCache.

    This class will back up the pytorch's random state during initialization. Then when the context is activated,
    the class will set up the random state with the backed-up one.
    returnNonec                b    t          j                    | _        t          | \  | _        | _        d S N)torchget_rng_statefwd_cpu_stater   fwd_gpu_devicesfwd_gpu_states)selftensorss     l/var/www/html/ai-engine/env/lib/python3.11/site-packages/sentence_transformers/losses/CachedGISTEmbedLoss.py__init__zRandContext.__init__   s.    "0224Ew4O1d111    c                    t           j                            | j        d          | _        | j                                         t          j        | j                   t          | j        | j	                   d S )NT)devicesenabled)
r   randomfork_rngr   _fork	__enter__set_rng_stater   r   r   r   s    r   r&   zRandContext.__enter__   sf    \**43GQU*VV

D.///$.0CDDDDDr   c                L    | j                             |||           d | _         d S r   )r%   __exit__)r   exc_typeexc_valexc_tbs       r   r*   zRandContext.__exit__#   s'    
Hgv666


r   N)r   r   )__name__
__module____qualname____doc__r   r&   r*    r   r   r   r      sb         P P P PE E E E     r   r   grad_outputr	   sentence_featuresIterable[dict[str, Tensor]]loss_objCachedGISTEmbedLossr   r   c           
        |j         J |j        J t          j                    5  t	          ||j         |j                  D ]\  }}}t	          |                    |dd|          |          D ]Y\  \  }}}}t          j        |                                |                                          | z  }	|	                                 Z	 ddd           dS # 1 swxY w Y   dS )zOA backward hook to backpropagate the cached gradients mini-batch by mini-batch.NTF)sentence_feature	with_gradcopy_random_staterandom_states)	cacher<   r   enable_gradzipembed_minibatch_iterdotflattenbackward)
r3   r4   r6   r9   gradr<   reps_mb_grad_mb	surrogates
             r   _backward_hookrI   (   sV    >%%%!---				 % %589JHN\d\r5s5s 	% 	%1dM,/--%5"&+"/	 .   - - 
% 
%(!Q "Igoo&7&79J9JKKkY	""$$$$
%	%% % % % % % % % % % % % % % % % % %s   B$CCCc                  h     e Zd Z	 	 	 d.d/ fdZd0dZ	 d1d2dZ	 d1d3d"Zd4d&Zd4d'Zd5d+Z	d6d-Z
 xZS )7r7   {Gz?    Fmodelr   guidetemperaturefloatmini_batch_sizeintshow_progress_barboolr   r   c                t   t                                                       t          |d         t                    rt	          d          || _        || _        || _        t          j	        d          | _
        t          |d         t                    rt          |d         t                    st	          d          t          j                    | _        || _        d| _        d| _        || _        |j        j        |j        j        k    p|j        |j        k     | _        | j        r|j        | _        dS dS )a  
        This loss is a combination of :class:`GISTEmbedLoss` and :class:`CachedMultipleNegativesRankingLoss`.
        Typically, :class:`MultipleNegativesRankingLoss` requires a larger batch size for better performance.
        :class:`GISTEmbedLoss` yields stronger training signals than :class:`MultipleNegativesRankingLoss` due to the
        use of a guide model for in-batch negative sample selection. Meanwhile, :class:`CachedMultipleNegativesRankingLoss`
        allows for scaling of the batch size by dividing the computation into two stages of embedding and loss
        calculation, which both can be scaled by mini-batches (https://arxiv.org/pdf/2101.06983.pdf).

        By combining the guided selection from :class:`GISTEmbedLoss` and Gradient Cache from
        :class:`CachedMultipleNegativesRankingLoss`, it is possible to reduce memory usage while maintaining performance
        levels comparable to those of :class:`GISTEmbedLoss`.

        Args:
            model: SentenceTransformer model
            guide: SentenceTransformer model to guide the in-batch negative sample selection.
            temperature: Temperature parameter to scale the cosine similarities.
            mini_batch_size: Mini-batch size for the forward pass, this denotes how much memory is actually used during
                training and evaluation. The larger the mini-batch size, the more memory efficient the training is, but
                the slower the training will be. It's recommended to set it as high as your GPU memory allows. The default
                value is 32.
            show_progress_bar: If True, a progress bar for the mini-batches is shown during training. The default is False.

        References:
            - Efficient Natural Language Response Suggestion for Smart Reply, Section 4.4: https://arxiv.org/pdf/1705.00652.pdf
            - Scaling Deep Contrastive Learning Batch Size under Memory Limited Setup: https://arxiv.org/pdf/2101.06983.pdf
            - GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning https://arxiv.org/abs/2402.16829

        Requirements:
            1. (anchor, positive) pairs or (anchor, positive, negative pairs)
            2. Should be used with large batch sizes for superior performance, but has slower training time than :class:`MultipleNegativesRankingLoss`

        Inputs:
            +-------------------------------------------------+--------+
            | Texts                                           | Labels |
            +=================================================+========+
            | (anchor, positive) pairs                        | none   |
            +-------------------------------------------------+--------+
            | (anchor, positive, negative) triplets           | none   |
            +-------------------------------------------------+--------+
            | (anchor, positive, negative_1, ..., negative_n) | none   |
            +-------------------------------------------------+--------+

        Recommendations:
            - Use ``BatchSamplers.NO_DUPLICATES`` (:class:`docs <sentence_transformers.training_args.BatchSamplers>`) to
              ensure that no in-batch negatives are duplicates of the anchor or positive samples.

        Relations:
            - Equivalent to :class:`GISTEmbedLoss`, but with caching that allows for much higher batch sizes

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                guide = SentenceTransformer("all-MiniLM-L6-v2")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.CachedGISTEmbedLoss(model, guide, mini_batch_size=64)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        r   zCachedGISTEmbedLoss is not compatible with a SentenceTransformer model based on a StaticEmbedding. Consider using GISTEmbedLoss instead.dimz_Both the training model and the guiding model must be based on the `transformers` architecture.N)superr   
isinstancer   
ValueErrorrM   rN   rO   r
   CosineSimilaritysimilarity_fctr   CrossEntropyLosscross_entropy_lossrQ   r=   r<   rS   	tokenizervocabmax_seq_lengthmust_retokenize)r   rM   rN   rO   rQ   rS   	__class__s         r   r   zCachedGISTEmbedLoss.__init__@   s8   \ 	eAh00 	8   

& 1b999%(K00 	
58[8Y8Y 	q   #%"5"7"7.04
=A!2O!U_%::ie>RUZUi>i 	  	-"_DNNN	- 	-r   embed1r	   embed2c                z    |                      |                    d          |                    d                    S )N   r   )r]   	unsqueeze)r   re   rf   s      r   
sim_matrixzCachedGISTEmbedLoss.sim_matrix   s4    ""6#3#3A#6#68H8H8K8KLLLr   Nr9   dict[str, Tensor]beginendr:   r;   random_stateRandContext | None!tuple[Tensor, RandContext | None]c                    |rt           nt          j        }|t                      n|}fd|                                D             }	|5   |            5  |rt	          |	                                 nd}                     |	          d         }
ddd           n# 1 swxY w Y   t          j                    5   j        r\ j        	                    |	d         d          } j
                            |          }	 fd|	                                D             }	 
                    |	          d         }ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   |
||fS )zYDo forward pass on a minibatch of the input features and return corresponding embeddings.Nc                ,    i | ]\  }}||         S r2   r2   ).0kvrl   rm   s      r   
<dictcomp>z7CachedGISTEmbedLoss.embed_minibatch.<locals>.<dictcomp>   s'    %[%[%[$!Qa59%[%[%[r   sentence_embedding	input_idsT)skip_special_tokensc                V    i | ]%\  }}||                     j        j                  &S r2   )torN   device)rs   keyvaluer   s      r   rv   z7CachedGISTEmbedLoss.embed_minibatch.<locals>.<dictcomp>   s=     2 2 2=GS%UXXdj&7882 2 2r   )r   r   no_graditemsr   valuesrM   rc   r`   batch_decoderN   tokenize)r   r9   rl   rm   r:   r;   rn   grad_contextrandom_state_contextsentence_feature_minibatchrepsdecoded
guide_repss   ` ``         r   embed_minibatchz#CachedGISTEmbedLoss.embed_minibatch   s    '0B{{U]0<0D{}}},%[%[%[%[%[BRBXBXBZBZ%[%[%["! 	Z 	Z T TTeo{,F,M,M,O,OPPkozz"<==>RST T T T T T T T T T T T T T T  	Z 	Z' "n992;?UY :  G 261D1DW1M1M.2 2 2 2KeKkKkKmKm2 2 2. "ZZ(BCCDXY
	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z		Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z 	Z Z--s[   E ;B E  B$	$E 'B$	(E >A?E	=E 	E	E E	E  E$'E$r<   list[RandContext] | None+Iterator[tuple[Tensor, RandContext | None]]c           
   #    K   |d         }|j         \  }}t          t          j        d|| j        d| j                             D ]?\  }}	|	| j        z   }
|                     ||	|
|||dn||                   \  }}}|||fV  @dS )z`Do forward pass on all the minibatches of the input features and yield corresponding embeddings.rx   r   zEmbed mini-batchesdescdisableN)r9   rl   rm   r:   r;   rn   )shape	enumeratetqdmtrangerQ   rS   r   )r   r9   r:   r;   r<   rx   bszrF   iber   r   rn   s                 r   r@   z(CachedGISTEmbedLoss.embed_minibatch_iter   s       -[9	QK$) 22  
 
 	1 	1DAq D((A-1-A-A!1#"3%2%:TTa@P .B . .*D*l 
L00000%	1 	1r   r   list[list[Tensor]]reps_guidedc                   t          |          t          |          k    rt          d          d |D             }d |D             }t          j        |d                             d                                                                        |d         j                  }|d         j        d         }g }t          j
        d|| j        d| j                   D ]}|| j        z   }	|                     |d         ||	         |d                   }
|                     |d         ||	         |d                   }|                     |d         ||	         |d                   }|
                    |                              d	d          }|                     |d         ||	         |d                   }|                     |d         ||	         |d                   }|                     |d         ||	         |d                   }t          j         ||
|k    <   t          j         |||k    <   t          j         |||k    <   t          j        |||gd
          }t          |          dk    rt%          dt          |                    D ]}|                     |d         ||	         ||                   }|                     |d         ||	         ||                   }t          j         |||k    <   t          j        ||gd
          }|| j        z  }|                     ||||	                   t          |          z  |z  }|                                 |                    |                                           t1          |                                          }d |D             | _        |S )zeGeneralized function to calculate the cross-entropy loss and cache the gradients wrt. the embeddings..reps and reps_guided must have the same lengthc                :    g | ]}t          j        |d           S r   rW   r   catrs   reps     r   
<listcomp>zJCachedGISTEmbedLoss.calculate_loss_and_cache_gradients.<locals>.<listcomp>   '    CCCsUYs222CCCr   c                :    g | ]}t          j        |d           S r   r   rs   	rep_guides     r   r   zJCachedGISTEmbedLoss.calculate_loss_and_cache_gradients.<locals>.<listcomp>   '    #]#]#]IEIiQ$?$?$?#]#]#]r   r   zPreparing cachesr   rh   offsetrV   rW      c                &    g | ]}d  |D             S )c                    g | ]	}|j         
S r2   )rD   )rs   rs     r   r   zUCachedGISTEmbedLoss.calculate_loss_and_cache_gradients.<locals>.<listcomp>.<listcomp>!  s    ***!qv***r   r2   )rs   rss     r   r   zJCachedGISTEmbedLoss.calculate_loss_and_cache_gradients.<locals>.<listcomp>!  s'    :::r**r***:::r   )lenr[   r   arangesizelongr{   r|   r   r   r   rQ   rS   rj   diagonalviewinfr   rangerO   r_   rC   appenddetachsumrequires_grad_r=   r   r   r   concatenated_repsconcatenated_guided_repslabels
batch_sizelossesr   r   guided_ap_simguided_aa_simguided_pp_sim
guided_simap_simaa_simpp_simscoresr   guided_neg_simneg_simloss_mbatchlosss                          r   "calculate_loss_and_cache_gradientsz6CachedGISTEmbedLoss.calculate_loss_and_cache_gradients   s   t99K((((MNNN DCdCCC#]#]Q\#]#]#] /277::;;@@BBEEFWXYFZFabb&q)/2
%' #..
 
 
 *	0 *	0A D((A !OO,DQ,G!,LNfghNijjM OO,DQ,G!,LNfghNijjM OO,DQ,G!,LNfghNijjM '//q/99>>r1EEJ __%6q%9!A#%>@QRS@TUUF__%6q%9!A#%>@QRS@TUUF__%6q%9!A#%>@QRS@TUUF 38)F=:-.27)F=:-.27)F=:-. Y7Q???F $%%))q#&7"8"899 A AA%)__5Ma5PQRSTQT5UWopqWr%s%sN"oo.?.B1Q3.GIZ[\I]^^G<AI:GNZ78"Y'8a@@@FF d..F(,(?(?qQRs(T(TWZ[aWbWb(beo(oK  """MM+,,..////6{{))++::T:::
r   c                   t          |          t          |          k    rt          d          d |D             }d |D             }t          j        |d                             d                                                                        |d         j                  }|d         j        d         }g }t          j
        d|| j        d| j                   D ]}|| j        z   }	|                     |d         ||	         |d                   }
|                     |d         ||	         |d                   }|                     |d         ||	         |d                   }|
                    |                              d	d          }|                     |d         ||	         |d                   }|                     |d         ||	         |d                   }|                     |d         ||	         |d                   }t          j         ||
|k    <   t          j         |||k    <   t          j         |||k    <   t          j        |||gd
          }t          |          dk    rt%          dt          |                    D ]}|                     |d         ||	         ||                   }|                     |d         ||	         ||                   }t          j         |||k    <   t          j        ||gd
          }|| j        z  }|                     ||||	                   t          |          z  |z  }|                    |           t-          |          }|S )zSGeneralized function to calculate the cross-entropy loss without caching gradients.r   c                :    g | ]}t          j        |d           S r   r   r   s     r   r   z6CachedGISTEmbedLoss.calculate_loss.<locals>.<listcomp>+  r   r   c                :    g | ]}t          j        |d           S r   r   r   s     r   r   z6CachedGISTEmbedLoss.calculate_loss.<locals>.<listcomp>,  r   r   r   zCalculating lossr   rh   r   rV   rW   r   )r   r[   r   r   r   r   r{   r|   r   r   r   rQ   rS   rj   r   r   r   r   r   rO   r_   r   r   r   s                          r   calculate_lossz"CachedGISTEmbedLoss.calculate_loss%  s   t99K((((MNNN DCdCCC#]#]Q\#]#]#] /277::;;@@BBEEFWXYFZFabb&q)/2
%' #..
 
 
 )	' )	'A D((A !OO,DQ,G!,LNfghNijjM OO,DQ,G!,LNfghNijjM OO,DQ,G!,LNfghNijjM '//q/99>>r1EEJ __%6q%9!A#%>@QRS@TUUF__%6q%9!A#%>@QRS@TUUF__%6q%9!A#%>@QRS@TUUF 38)F=:-.27)F=:-.27)F=:-. Y7Q???F $%%))q#&7"8"899 A AA%)__5Ma5PQRSTQT5UWopqWr%s%sN"oo.?.B1Q3.GIZ[\I]^^G<AI:GNZ78"Y'8a@@@FF d..F(,(?(?qQRs(T(TWZ[aWbWb(beo(oKMM+&&&&6{{r   r4   r5   r   c                   g }g }g | _         |D ]}g }g }g }|                     |dd          D ]{\  }	}
}|                    |	                                                                           |                    |
                                           |                    |           ||                    |           |                    |           | j                             |           t          j                    rA|                     ||          }|                    t          t          ||                      n|                     ||          }|S )NFT)r9   r:   r;   )r4   r6   )r<   r@   r   r   r   r   is_grad_enabledr   register_hookr   rI   r   )r   r4   r   r   r   r9   reps_mbsreps_guided_mbsrandom_state_mbsrE   reps_guided_mbrn   r   s                r   forwardzCachedGISTEmbedLoss.forward`  sw    1 	8 	8H O!9=9R9R!1"& :S : : 6 65
  0 0 ? ? A ABBB&&~'<'<'>'>??? ''5555KK!!!///%%&67777 "" 	:::4MMD w~IZeijjjkkkk &&t[99Dr   dict[str, Any]c                     | j         | j        dS )NrN   rO   r   r(   s    r   get_config_dictz#CachedGISTEmbedLoss.get_config_dict  s    Z+
 
 	
r   )rK   rL   F)rM   r   rN   r   rO   rP   rQ   rR   rS   rT   r   r   )re   r	   rf   r	   r   r	   r   )r9   rk   rl   rR   rm   rR   r:   rT   r;   rT   rn   ro   r   rp   )
r9   rk   r:   rT   r;   rT   r<   r   r   r   )r   r   r   r   r   r	   )r4   r5   r   r	   r   r	   )r   r   )r.   r/   r0   r   rj   r   r@   r   r   r   r   __classcell__)rd   s   @r   r7   r7   ?   s        
 "!"'e- e- e- e- e- e- e-NM M M M ,0. . . . .F 371 1 1 1 1<= = = =~9 9 9 9v   @
 
 
 
 
 
 
 
r   )r3   r	   r4   r5   r6   r7   r   r   )
__future__r   collections.abcr   r   
contextlibr   	functoolsr   typingr   r   r   r	   r
   torch.utils.checkpointr   r   sentence_transformersr   sentence_transformers.modelsr   r   r   rI   Moduler7   r2   r   r   <module>r      sM   " " " " " " . . . . . . . . " " " " " "                       G G G G G G G G 5 5 5 5 5 5 E E E E E E E E       .% % % %.E
 E
 E
 E
 E
") E
 E
 E
 E
 E
r   