
    Ng-                         d dl mZ d dlZd dlmZ d dlmZmZ d dlmZ d Z G d dej                  Z	 G d	 d
ej                  Z
dS )    )OptionalN)nn)r   Tensor)_get_activation_fnc                    t          | d          rOt          | d          r?t          j                    | _        | `| j        }| j        }t          ||          | _        nt          | d          rOt          | d          r?t          j                    | _        | `| j        }| j        }t          ||          | _        nqd| 	                                v sd| 	                                v r'| `
| j        }| j        }t          ||          | _
        nt          d           t          d           t          | d	          rd
| _        | S )Nglobal_poolfc)num_classesinitial_num_features
classifierRegNetTResNetz;Model code-writing is not aligned currently with ml-decoder	drop_rater   )hasattrr   Identityr   r	   r
   num_features	MLDecoderr   	_get_nameheadprintexitr   )modelr
   r   s      R/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/ml_decoder.pyadd_ml_decoder_headr   	   sJ   um$$ )=)= KMMH')<XXX		&	& 75,+G+G KMM')$S_```	U__&&	&	&)u7H7H*H*HJ');\ZZZ

KLLLRuk"" L    c                        e Zd Z	 	 d	 d fdZ fd	Z	 	 	 	 dd
ededee         dee         dee         dee         defdZ xZS )TransformerDecoderLayerOptimal      皙?reluh㈵>returnNc                    t          t          |                                            t          j        ||          | _        t          j        |          | _        t          j        |          | _        t          j        |          | _	        t          j        |          | _
        t          j        |||          | _        t          j        ||          | _        t          j        ||          | _        t          j        ||          | _        t          j        ||          | _        t%          |          | _        d S )N)eps)dropout)superr   __init__r   	LayerNormnorm1Dropoutr'   dropout1dropout2dropout3MultiheadAttentionmultihead_attnLinearlinear1linear2norm2norm3r   
activation)selfd_modelnheaddim_feedforwardr'   r7   layer_norm_eps	__class__s          r   r)   z'TransformerDecoderLayerOptimal.__init__$   s    ,d33<<>>>\'~>>>
z'**
7++
7++
7++ 3GUGTTT y/::y'::\'~>>>
\'~>>>
,Z88r   c                     d|vrt           j        j        j        |d<   t	          t
          |                               |           d S )Nr7   )torchr   
functionalr"   r(   r   __setstate__)r8   stater=   s     r   rA   z+TransformerDecoderLayerOptimal.__setstate__8   sE    u$$"'("5":E,,d33@@GGGGGr   tgtmemorytgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskc           	         ||                      |          z   }|                     |          }|                     |||          d         }||                     |          z   }|                     |          }|                     |                     |                     |                     |                                        }|| 	                    |          z   }| 
                    |          }|S )Nr   )r-   r+   r1   r.   r5   r4   r'   r7   r3   r/   r6   )r8   rC   rD   rE   rF   rG   rH   tgt2s           r   forwardz&TransformerDecoderLayerOptimal.forward=   s     DMM#&&&jjoo""377:DMM$'''jjoo||DLLc9J9J)K)KLLMMDMM$'''jjoo
r   )r   r    r!   r"   r#   )r$   N)NNNN)	__name__
__module____qualname__r)   rA   r   r   rK   __classcell__r=   s   @r   r   r   #   s        W] $9)-9 9 9 9 9 9(H H H H H
 QU049=<@ 6 6 Xf=M %f-&.v&6 *2&)9 FL       r   r   c                   &     e Zd Zd fd	Zd Z xZS )r   r      r    c                 l   t          t          |                                            |dk     rdn|}||k    r|}|| _        |dk     rdn|}t	          j        ||          | _        d}d}d}t          |||          }	t	          j        |	|          | _	        t	          j
        ||          | _        | j                            d	           || _        t          ||z  d
z             | _        t           j                            t!          j        ||| j                            | _        t           j                            t!          j        |                    | _        t           j        j                            | j                   t           j        j                            | j        d           d S )Nr   d   rR   r!      r    )r9   r;   r'   )
num_layersFg+?)r(   r   r)   embed_len_decoderr   r2   embed_standartr   TransformerDecoderdecoder	Embeddingquery_embedrequires_grad_r
   intduplicate_factorr?   	Parameterr   duplicate_poolingduplicate_pooling_biasinitxavier_normal_	constant_)r8   r
   num_of_groupsdecoder_embeddingr   rW   decoder_dropoutnum_layers_decoderr;   layer_decoder=   s             r   r)   zMLDecoder.__init__[   s   i'')))#01#4#4CC-{** +!2 $5q#8#8CC>O i(<>OPP 5>OFU_np p p,\FXYYY <(9;LMM''... ' #K2C$Ce$K L L!&!3!3L*,=t?TUU"W "W&+h&8&8k9R9R&S&S#$$T%;<<< ;Q?????r   c                    t          |j                  dk    r*|                    d                              dd          }n|}|                     |          }t
          j        j                            |d          }|j        d         }| j	        j
        }|                    d                              d|d          }|                     ||                    dd                    }|                    dd          }t          j        |j        d         |j        d         | j        |j        |j                  }t%          | j                  D ]D}	|d d |	d d f         }
| j        |	d d d d f         }t          j        |
|          |d d |	d d f<   E|                    d          d d d | j        f         }|| j        z  }|}|S )	N      rU   T)inplacer   r   )devicedtype)lenshapeflatten	transposerX   r?   r   r@   r"   r\   weight	unsqueezeexpandrZ   zerosr_   ro   rp   rangerW   ra   matmulr
   rb   )r8   xembedding_spatialembedding_spatial_786bsr\   rC   h
out_extrapih_iw_ih_outlogitss                 r   rK   zMLDecoder.forward{   s   qw<<1 !		! 6 6q! < < ! $ 3 34E F F % 3 8 89NX\ 8 ] ]"(+&-##A&&--b"b99LL3==aCCDDKK1[QWQZ9NWXW_ghgnooo
t-.. 	9 	9AAAAq!!!G*C(AAAqqq1C"',sC"8"8Jqqq!QQQw""1%%aaa):$*:):&:;,,r   )r   rR   r    )rL   rM   rN   r)   rK   rO   rP   s   @r   r   r   Z   sS        @ @ @ @ @ @@      r   r   )typingr   r?   r   r   torch.nn.modules.transformerr   r   Moduler   r    r   r   <module>r      s                         ; ; ; ; ; ;  4& & & & &RY & & &n8 8 8 8 8	 8 8 8 8 8r   