
    Ng-                         d Z ddlmZmZmZ ddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZmZ dd	lmZ  G d
 dej                  Z G d dej                  ZdS )a:   Attention Pool 2D

Implementations of 2D spatial feature pooling using multi-head attention instead of average pool.

Based on idea in CLIP by OpenAI, licensed Apache 2.0
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

Hacked together by / Copyright 2021 Ross Wightman
    )OptionalUnionTupleN   )use_fused_attn)	to_2tuple)resample_abs_pos_embed)apply_rot_embedRotaryEmbedding)trunc_normal_c                   T    e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 	 d!d
e	de
e	         dee	ee	e	f         f         de
e	         de
e	         de
e	         dededededef fdZd"defdZd#de
e	         de
e         fdZdej        de	de	dej        fdZd"defd Z xZS )$RotAttentionPool2daB   Attention based 2D feature pooling w/ rotary (relative) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    Adapted from the AttentionPool2d in CLIP w/ rotary embedding instead of learned embed.
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: While this impl does not require a fixed feature size, performance at differeing resolutions from
    train varies widely and falls off dramatically. I'm not sure if there is a way around this... -RW
    
fused_attnN   @   TFtoken        in_featuresout_featuresref_feat_size	embed_dimhead_dim	num_headsqkv_biasqkv_separate	pool_typeclass_token	drop_ratec                    t                                                       |	dv sJ |p|x| _        }|| _        |p|| _        t          |          }|||z  dk    sJ ||z  }n||z  dk    sJ ||z  }|| _        || _        |	                                | _	        | j        dz  | _
        t                      | _        |
r-t          j        t          j        d|                    | _        nd | _        |r\t          j        |||          | _        t          j        |||          | _        t          j        |||          | _        d | _        nt          j        ||dz  |          | _        t          j        |          | _        t          j        || j                  | _        t5          | j        d|          | _        d S )	N r   r         r   bias   F)	in_pixelsref_feat_shape)super__init__r   r   r   r   r   r   lowerr   scaler   r   nn	Parametertorchzeros	cls_tokenLinearqkvqkvDropoutdropprojr   	pos_embed)selfr   r   r   r   r   r   r   r   r   r   r   	__class__s               X/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/attention_pool2d.pyr)   zRotAttentionPool2d.__init__"   s    	M))))%.%=+=&(7K!-00 y(A---- I-HHx'1,,,,!X-I" "**]d*
(** 	"\%+a*C*CDDDNN!DN 	LY{IHEEEDFY{IHEEEDFY{IHEEEDFDHHyi!m(KKKDHJy))	Ii):;;	(%Xefff    zero_init_lastc                    | j         | j        j        }t          | j        j        |dz             t
          j                            | j        j                   t          | j	        j        |dz             t
          j                            | j	        j                   t          | j
        j        |dz             t
          j                            | j
        j                   d S | j         j        }t          | j         j        |dz             t
          j                            | j         j                   d S Nr"   )std)r5   r2   r   r   weightr,   initzeros_r$   r3   r4   r:   r>   r   s      r<   init_weightszRotAttentionPool2d.init_weightsR   s    8&,K$&-[D-@AAAAGNN46;'''$&-[D-@AAAAGNN46;'''$&-[D-@AAAAGNN46;'''''(.K$(/{d/BCCCCGNN48=)))))r=   num_classesc                     ||dv sJ || _         |N|dk    rt          j        | j        |          nt          j                    | _        |dk    r|n| j        | _        d S d S Nr    r   r   r,   r1   r   Identityr8   r   r   r:   rG   r   s      r<   resetzRotAttentionPool2d.reset`   x     ----&DN"DORSOO	$"2K@@@Y[YdYfYfDI/:QDND #"r=   xHWreturnc                     | j         dk    r|d d df         }nE|d d dd f                             |j        d         ||d                              dddd          }|S Nr   r   r   r%      r   reshapeshapepermuter:   rO   rP   rQ   s       r<   _poolzRotAttentionPool2d._pooli   l    >W$$!!!Q$AA !!!QRR%  Q266>>q!QJJAr=   
pre_logitsc                    |j         \  }}}}||z  }|                    d                              dd          }| j        .t	          j        |                    dd          |gd          }n=t	          j        | j                            |j         d         dd          |gd          }| j        | 	                    |          
                    ||dz   | j        | j                                      dd          }|                     |          
                    ||dz   | j        | j                                      dd          }	|                     |          
                    ||dz   | j        | j                                      dd          }
ni|                     |          
                    ||dz   d| j        | j                                      ddddd	          }|                    d          \  }}	}
| j                            ||f          \  }}t	          j        |d d d d d dd d f         t'          |d d d d dd d d f         ||          gd                              |
          }t	          j        |	d d d d d dd d f         t'          |	d d d d dd d d f         ||          gd                              |
          }	| j        r"t,          j                            ||	|
          }n>|| j        z  }||	                    d
d          z  }|                    d          }||
z  }|                    dd          
                    ||dz   d          }|                     |          }|r|                     |||          }|S |                     |          }|                     |||          }|S )NrV   r   Tkeepdimdimr   rU   r%      )rY   flatten	transposer0   r.   catmeanexpandr5   r2   rX   r   r   r3   r4   rZ   unbindr9   	get_embedr
   type_asr   r,   
functionalscaled_dot_product_attentionr+   softmaxr7   r\   r8   )r:   rO   r^   B_rP   rQ   Nr2   r3   r4   rserceattns                 r<   forwardzRotAttentionPool2d.forwardq   s   W
1aEIIaLL""1a((>!	166!T622A6A>>>AA	4>00RDDaHaPPPA8q		!!!QUDNDMJJTTUVXYZZAq		!!!QUDNDMJJTTUVXYZZAq		!!!QUDNDMJJTTUVXYZZAA##Aq1uaOOWWXY[\^_abdeffAhhqkkGAq!>++QF33SIqAAArr111~qAAAqrr111~sC'P'PQWXYYYaabcddIqAAArr111~qAAAqrr111~sC'P'PQWXYYYaabcdd? 	::1aCCAADJAq{{2r***D<<B<''DqAKK1%%aQ33IIaLL 	

1a##AHIIaLLJJq!Qr=   )
Nr   Nr   NTFr   Fr   FNN)__name__
__module____qualname____doc__r.   jitFinalbool__annotations__intr   r   r   strfloatr)   rF   rM   Tensorr\   rw   __classcell__r;   s   @r<   r   r      s          	%%%%
 +/9:'+&('+!!&$ %!.g .g.g #3-.g !eCHo!56	.g
  }.g sm.g  }.g .g .g .g .g .g .g .g .g .g .g`* *4 * * * *S S# S(3- S S S Su|       " "T " " " " " " " "r=   r   c                   T    e Zd ZU dZej        j        e         ed<   	 	 	 	 	 	 	 	 	 	 d!d
e	de
e	ee	e	f         f         dee	         dee	         dee	         dee	         dededededef fdZd"defdZd#dee	         dee         fdZdej        de	de	dej        fdZd"defd Z xZS )$AttentionPool2da   Attention based 2D feature pooling w/ learned (absolute) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    It was based on impl in CLIP by OpenAI
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: This requires feature size upon construction and well prevent adaptive sizing of the network.
    r   r   Nr   TFr   r   r   	feat_sizer   r   r   r   r   r   r   r   r   c                 6   t                                                       |	dv sJ |p|x| _        }|| _        |p|| _        |||z  dk    sJ ||z  }n||z  dk    sJ ||z  }t          |          | _        | j        d         | j        d         z  | _        || _        || _	        |	| _
        | j	        dz  | _        t                      | _        |
r-t          j        t!          j        d|                    | _        nd | _        |r\t          j        |||          | _        t          j        |||          | _        t          j        |||          | _        d | _        n4d x| _        x| _        | _        t          j        ||dz  |          | _        t          j        |          | _        t          j        || j                  | _        t          j        t!          j        | j        dz   |                    | _        |                                  d S )Nr    r   r   r"   r#   r%   )r(   r)   r   r   r   r   r   seq_lenr   r   r   r+   r   r   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   rF   )r:   r   r   r   r   r   r   r   r   r   r   r   r;   s               r<   r)   zAttentionPool2d.__init__   s    	M))))%.%=+=&(7K y(A---- I-HHx'1,,,,!X-I"9--~a(4>!+<<" "]d*
(** 	"\%+a*C*CDDDNN!DN 	LY{IHEEEDFY{IHEEEDFY{IHEEEDFDHH'++DF+TVdfyi!m(KKKDHJy))	Ii):;;	ek$,2BK&P&PQQr=   r>   c                    | j         | j        j        }t          | j        j        |dz             t
          j                            | j        j                   t          | j	        j        |dz             t
          j                            | j	        j                   t          | j
        j        |dz             t
          j                            | j
        j                   nS| j         j        }t          | j         j        |dz             t
          j                            | j         j                   t          | j        |dz             d S r@   )r5   r2   r   r   rB   r,   rC   rD   r$   r3   r4   r9   rE   s      r<   rF   zAttentionPool2d.init_weights   s   8&,K$&-[D-@AAAAGNN46;'''$&-[D-@AAAAGNN46;'''$&-[D-@AAAAGNN46;''''(.K$(/{d/BCCCCGNN48=)))dn+*=>>>>>>r=   rG   c                     ||dv sJ || _         |N|dk    rt          j        | j        |          nt          j                    | _        |dk    r|n| j        | _        d S d S rI   rJ   rL   s      r<   rM   zAttentionPool2d.reset   rN   r=   rO   rP   rQ   rR   c                     | j         dk    r|d d df         }nE|d d dd f                             |j        d         ||d                              dddd          }|S rT   rW   r[   s       r<   r\   zAttentionPool2d._pool   r]   r=   r^   c                 F   |j         \  }}}}||z  }|                    d                              dd          }| j        .t	          j        |                    dd          |gd          }n=t	          j        | j                            |j         d         dd          |gd          }t          | j	        
                    d          ||fd          }||z   }| j        |                     |                              ||dz   | j        | j                                      dd          }	|                     |                              ||dz   | j        | j                                      dd          }
|                     |                              ||dz   | j        | j                                      dd          }nf|                     |                              |dd	| j        | j                                      ddd	dd
          }|                    d          \  }	}
}| j        r"t*          j                            |	|
|          }n>|	| j        z  }	|	|
                    dd          z  }|                    d          }||z  }|                    dd                              ||dz   d          }|                     |          }|r|                     |||          }|S |                     |          }|                     |||          }|S )NrV   r   Tr`   rb   r   rU   )num_prefix_tokensr%   rd   re   )rY   rf   rg   r0   r.   rh   ri   rj   r	   r9   	unsqueezer5   r2   rX   r   r   r3   r4   rZ   rk   r   r,   rn   ro   r+   rp   r7   r\   r8   )r:   rO   r^   rq   rr   rP   rQ   rs   r9   r2   r3   r4   rv   s                r<   rw   zAttentionPool2d.forward   s   W
1aEIIaLL""1a((>!	166!T622A6A>>>AA	4>00RDDaHaPPPA*4>+C+CA+F+FAbcddd		M8q		!!!QUDNDMJJTTUVXYZZAq		!!!QUDNDMJJTTUVXYZZAq		!!!QUDNDMJJTTUVXYZZAA##Ar1dndmLLTTUVXY[\^_abccAhhqkkGAq!? 	::1aCCAADJAq{{2r***D<<B<''DqAKK1%%aQ33IIaLL 	

1a##AHIIaLLJJq!Qr=   )
r   NNr   NTFr   Fr   rx   ry   )rz   r{   r|   r}   r.   r~   r   r   r   r   r   r   r   r   r   r)   rF   rM   r   r\   rw   r   r   s   @r<   r   r      s          	%%%%
 67*.'+&('+!!&$ %!2 22 S%S/122 #3-	2
  }2 sm2  }2 2 2 2 2 2 2 2 2 2 2h? ?4 ? ? ? ?S S# S(3- S S S Su|       ! !T ! ! ! ! ! ! ! !r=   r   )r}   typingr   r   r   r.   torch.nnr,   configr   helpersr   r9   r	   pos_embed_sincosr
   r   weight_initr   Moduler   r    r=   r<   <module>r      s    * ) ) ) ) ) ) ) ) )        " " " " " "       - - - - - - > > > > > > > > & & & & & &} } } } } } } }@@ @ @ @ @bi @ @ @ @ @r=   