
    Ng                         d dl mZ d dlZd dlmZ d dlmc mZ ddlm	Z	 ddl
mZ ddlmZ  G d dej                  ZdS )	    )OptionalN   )use_fused_attn)Mlp)trunc_normal_tf_c                        e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 dde	de	de	de	de
e	         dededede	de	dedede
ej                 def fdZd Zd Z xZS )AttentionPoolLatentz' Attention pooling w/ latent query
    
fused_attnN         @TFr    token        in_featuresout_features	embed_dim	num_heads	feat_size	mlp_ratioqkv_biasqk_norm
latent_len
latent_dim	pos_embed	pool_type
norm_layerdropc                 0   t                                                       |p|}|p|}||z  dk    sJ || _        ||z  | _        || _        | j        dz  | _        || _        t                      | _        |dk    r1|J t          j
        t          j        ||                    | _        nd | _        |
p|| _        |	| _        t          j
        t          j        d| j        |                    | _        t          j        |||          | _        t          j        ||dz  |          | _        |r || j                  nt          j                    | _        |r || j                  nt          j                    | _        t          j        ||          | _        t          j        |          | _        | ||          nt          j                    | _        t7          |t9          ||z                      | _        |                                  d S )Nr         absr   )bias   )super__init__r   head_dimr   scalepoolr   r
   nn	Parametertorchzerosr   r   r   latentLinearqkvIdentityq_normk_normprojDropout	proj_dropnormr   intmlpinit_weights)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   	__class__s                  V/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/attention_pool.pyr$   zAttentionPoolLatent.__init__   s   " 	,	#2{9$))))"!Y."]d*
	(**(((\%+i*M*MNNDNN!DN$1	$l5;q$/9#M#MNN9ih???)Iy1}8DDD3:Mjj///3:Mjj///Ii33	D))0:0FJJ|,,,BKMM	y#i)&;"<"<==    c                     | j         )t          | j         | j         j        d         dz             t          | j        | j        dz             d S )Nr   r   )std)r   r   shaper,   r   )r:   s    r<   r9   z AttentionPoolLatent.init_weightsC   sP    >%T^1Ea1HD1PQQQQ$/T*ABBBBBBr=   c                    |j         \  }}}| j        5|| j                            d                              |j                  z   }| j                            |dd          }|                     |                              || j	        | j
        | j                                      dd          }|                     |                              ||d| j
        | j                                      ddddd          }|                    d          \  }}	|                     |          |                     |          }}| j        rt'          j        |||	          }n>|| j        z  }||                    dd          z  }
|
                    d          }
|
|	z  }|                    dd                              || j	        |          }|                     |          }|                     |          }||                     |                     |                    z   }| j        d	k    r|d d df         }n | j        d
k    r|                    d          }|S )Nr   r   r"         )dimr   avg)r@   r   	unsqueezetodtyper,   expandr.   reshaper   r   r%   	transposer/   permuteunbindr1   r2   r
   Fscaled_dot_product_attentionr&   softmaxr3   r5   r8   r6   r'   mean)r:   xBNCq_latentr.   r/   kvattns              r<   forwardzAttentionPoolLatent.forwardH   s   '1a>%DN,,Q//2217;;;A;%%aR00FF8$$QWWaabcefggWWQZZ1aGGOOPQSTVWYZ\]^^yy||1{{1~~t{{1~~1? 	.q!Q77AADJAq{{2r***D<<B<''DqAKK1%%a!<<IIaLLNN11&&& 9!!!Q$AAY%q		Ar=   )NNr   Nr   TFr   Nr   r   Nr   )__name__
__module____qualname____doc__r*   jitFinalbool__annotations__r7   r   floatstrr(   Moduler$   r9   r\   __classcell__)r;   s   @r<   r	   r	      sC         	%%%%
 !%!'+"!!"$.20 00 0 	0
 0  }0 0 0 0 0 0 0 0 !+0 0 0 0 0 0 0dC C C
! ! ! ! ! ! !r=   r	   )typingr   r*   torch.nnr(   torch.nn.functional
functionalrP   configr   r8   r   weight_initr   rg   r	    r=   r<   <module>rp      s                          " " " " " "       ) ) ) ) ) )] ] ] ] ]") ] ] ] ] ]r=   