
    NgC                     2   d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	c mZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZmZ d	gZ G d
 de	j                   Z! G d de	j                   Z" G d de	j                   Z# G d de	j                   Z$ G d de	j                   Z% G d d	e	j                   Z&d Z'd%dZ(d&dZ) e e)d           e)d           e)d           e)d           e)d           e)d           e)d          d          Z*ed%de&fd            Z+ed%de&fd            Z,ed%de&fd             Z-ed%de&fd!            Z.ed%de&fd"            Z/ed%de&fd#            Z0ed%de&fd$            Z1dS )'a   Pyramid Vision Transformer v2

@misc{wang2021pvtv2,
      title={PVTv2: Improved Baselines with Pyramid Vision Transformer},
      author={Wenhai Wang and Enze Xie and Xiang Li and Deng-Ping Fan and Kaitao Song and Ding Liang and
        Tong Lu and Ping Luo and Ling Shao},
      year={2021},
      eprint={2106.13797},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

Based on Apache 2.0 licensed code at https://github.com/whai362/PVT

Modifications and timm support by / Copyright 2022, Ross Wightman
    N)CallableListOptionalUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath	to_2tuple	to_ntupletrunc_normal_	LayerNormuse_fused_attn   )build_model_with_cfg)register_modelgenerate_default_cfgsPyramidVisionTransformerV2c                   L     e Zd Zddej        ddf fd	Zdee         fdZ xZ	S )MlpWithDepthwiseConvN        Fc           	         t                                                       |p|}|p|}t          j        ||          | _        |rt          j                    nt          j                    | _        t          j        ||dddd|          | _	         |            | _
        t          j        ||          | _        t          j        |          | _        d S )N   r   T)biasgroups)super__init__nnLinearfc1ReLUIdentityreluConv2ddwconvactfc2Dropoutdrop)selfin_featureshidden_featuresout_features	act_layerr)   
extra_relu	__class__s          N/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/pvt_v2.pyr   zMlpWithDepthwiseConv.__init__#   s     	#2{)8[9[/::!+>BGIII	i!QPT]lmmm9;;9_l;;Jt$$			    	feat_sizec                    |                      |          }|j        \  }}}|                    dd                              |||d         |d                   }|                     |          }|                     |          }|                    d                              dd          }|                     |          }|                     |          }| 	                    |          }|                     |          }|S )Nr      r   )
r    shape	transposeviewr#   r%   flattenr&   r)   r'   )r*   xr3   BNCs         r1   forwardzMlpWithDepthwiseConv.forward6   s    HHQKK'1aKK1""1a1y|DDIIaLLKKNNIIaLL""1a((HHQKKIIaLLHHQKKIIaLLr2   )
__name__
__module____qualname__r   GELUr   r   intr>   __classcell__r0   s   @r1   r   r   "   so         !g% % % % % %&DI        r2   r   c                   p     e Zd ZU ej        j        e         ed<   	 	 	 	 	 	 d
 fd	Zde	e
         fd	Z xZS )	Attention
fused_attn   r   FTr   c                    t                                                       ||z  dk    sJ d| d| d            || _        || _        ||z  | _        | j        dz  | _        t                      | _        t          j	        |||          | _
        t          j	        ||dz  |          | _        t          j        |          | _        t          j	        ||          | _        t          j        |          | _        |s[d | _        |dk    r7t          j        ||||	          | _        t          j        |          | _        nd | _        d | _        d | _        d S t          j        d
          | _        t          j        ||dd	          | _        t          j        |          | _        t          j                    | _        d S )Nr   zdim z  should be divided by num_heads .g      )r   r5   r   )kernel_sizestride   )r   r   dim	num_headshead_dimscaler   rH   r   r   qkvr(   	attn_dropproj	proj_droppoolr$   srr   normr&   AdaptiveAvgPool2drB   )	r*   rO   rP   sr_ratiolinear_attnqkv_biasrU   rW   r0   s	           r1   r   zAttention.__init__G   s    	Y!###%]C%]%]QZ%]%]%]###"y(]d*
(**3(333)Cqx888I..Ic3''	I.. 	!DI!||)C(8TTTL--		 	DHHH,Q//DIiSaBBBDGS))DIwyyDHHHr2   r3   c                    |j         \  }}}|\  }}|                     |                              ||| j        d                              dddd          }| j        |                    ddd                              ||||          }|                     |                     |                                        ||d                              ddd          }|                     |          }|                     |          }| 	                    |                              |dd| j        | j
                                      ddddd          }	n#| j        |                    ddd                              ||||          }|                     |                              ||d                              ddd          }|                     |          }| 	                    |                              |dd| j        | j
                                      ddddd          }	nM| 	                    |                              |dd| j        | j
                                      ddddd          }	|	                    d          \  }
}| j        r,t          j        ||
|| j        r| j        j        nd          }nS|| j        z  }||
                    d	d          z  }|                    d
          }|                     |          }||z  }|                    dd                              |||          }|                     |          }|                     |          }|S )Nr   r5   r   r      r   )	dropout_prO   )r6   rS   reshaperP   permuterX   rY   rZ   r&   rT   rQ   unbindrH   Fscaled_dot_product_attentiontrainingrU   prR   r7   softmaxrV   rW   )r*   r:   r3   r;   r<   r=   HWrS   rT   kvattns                r1   r>   zAttention.forwardo   s   '1a1FF1IIaDNB77??1aKK9 		!Q""**1aA66A		!%%--aB77??1aHHA		!AA##Ar1dndmLLTTUVXY[\^_abccBBw"IIaA&&..q!Q::GGAJJ&&q!R0088AqAAIIaLLWWQZZ''2q$.$-PPXXYZ\]_`bcefggWWQZZ''2q$.$-PPXXYZ\]_`bcefggyy||1? 	.q!QVZVcBk$.BRBRiklllAADJAq{{2r***D<<B<''D>>$''DqAKK1%%aA..IIaLLNN1r2   )rI   r   FTr   r   )r?   r@   rA   torchjitFinalbool__annotations__r   r   rC   r>   rD   rE   s   @r1   rG   rG   D   s         	%%%%
 &! &! &! &! &! &!P!DI ! ! ! ! ! ! ! !r2   rG   c            	       T     e Zd Zdddddddej        ef	 fd	Zdee         fdZ	 xZ
S )Block      @r   Fr   c           	         t                                                        ||          | _        t          |||||||          | _        |	dk    rt          |	          nt          j                    | _         ||          | _	        t          |t          ||z            |
||          | _        |	dk    rt          |	          nt          j                    | _        d S )N)rP   r\   r]   r^   rU   rW   r   )r+   r,   r.   r)   r/   )r   r   norm1rG   rq   r
   r   r"   
drop_path1norm2r   rC   mlp
drop_path2)r*   rO   rP   	mlp_ratior\   r]   r^   rW   rU   	drop_pathr.   
norm_layerr0   s               r1   r   zBlock.__init__   s     	Z__
#
 
 
	 2;R(9---R[]]Z__
'i00"
 
 
 2;R(9---R[]]r2   r3   c                    ||                      |                     |                     |          |                    z   }||                     |                     |                     |          |                    z   }|S N)r|   rq   r{   r   r~   r}   )r*   r:   r3   s      r1   r>   zBlock.forward   sc    		$**Q-- C CDDDA	 B BCCCr2   )r?   r@   rA   r   rB   r   r   r   rC   r>   rD   rE   s   @r1   rx   rx      s         g #S #S #S #S #S #SJDI        r2   rx   c                   *     e Zd ZdZd fd	Zd Z xZS )	OverlapPatchEmbedz Image to Patch Embedding
    rN   ra   r      c           	      D   t                                                       t          |          }t          |          |k    s
J d            || _        t          j        |||||d         dz  |d         dz  f          | _        t          j        |          | _	        d S )Nz!Set larger patch_size than strider   r5   r   )rM   padding)
r   r   r   max
patch_sizer   r$   rV   r   rZ   )r*   r   rM   in_chans	embed_dimr0   s        r1   r   zOverlapPatchEmbed.__init__   s    z**
:''')L'''$IiJqMQ$6
18J#KM M M	 L++			r2   c                     |                      |          }|                    dddd          }|                     |          }|S )Nr   r5   r   r   )rV   rf   rZ   r*   r:   s     r1   r>   zOverlapPatchEmbed.forward   s<    IIaLLIIaAq!!IIaLLr2   )rN   ra   r   r   )r?   r@   rA   __doc__r   r>   rD   rE   s   @r1   r   r      sV         , , , , , ,      r2   r   c                        e Zd Zdddddddddef
deded	ed
ededededededededeee         ef         de	f fdZ
d Z xZS )PyramidVisionTransformerStageTrI   r   Fry   r   rO   dim_outdepth
downsamplerP   r\   r]   r   r^   rW   rU   r   r   c                 T  	
 t                                                       d| _        |rt          dd|          | _        n|k    sJ d | _        t          j        
	f
dt          |          D                       | _                   | _	        d S )NFr   r5   r   rM   r   r   c                 |   
 g | ]8}t          
	t          t                    r|         n 
  
        9S ))
rO   rP   r\   r]   r   r^   rW   rU   r   r   )rx   
isinstancelist).0irU   r   r   r]   r   r   rP   rW   r^   r\   s     r1   
<listcomp>z:PyramidVisionTransformerStage.__init__.<locals>.<listcomp>   so     %! %! %!  &+#&0D&A&APilly!&
 &
 &
 %! %! %!r2   )
r   r   grad_checkpointingr   r   r   
ModuleListrangeblocksrZ   )r*   rO   r   r   r   rP   r\   r]   r   r^   rW   rU   r   r   r0   s     `  `````````r1   r   z&PyramidVisionTransformerStage.__init__   s      	"' 		#/!	  DOO '>>>>"DOm %! %! %! %! %! %! %! %! %! %! %! %! %! <<%! %! %! " " Jw''			r2   c                    | j         |                      |          }|j        \  }}}}||f}|                    |d|          }| j        D ]J}| j        r5t
          j                                        st          j        |||          }> |||          }K| 	                    |          }|                    ||d         |d         d          
                    dddd                                          }|S )Nr`   r   r   r   r5   )r   r6   re   r   r   rr   rs   is_scripting
checkpointrZ   rf   
contiguous)r*   r:   r;   rm   rn   r=   r3   blks           r1   r>   z%PyramidVisionTransformerStage.forward  s    ?&""AW
1aF	IIaQ; 	& 	&C& &uy/E/E/G/G &)#q)<<C9%%IIaLLIIa1y|R88@@Aq!LLWWYYr2   )r?   r@   rA   r   rC   ru   floatr   r   r   r   r>   rD   rE   s   @r1   r   r      s          $ %"!!!36#,+( +(+( +( 	+(
 +( +( +( +( +( +( +( +( T%[%/0+( !+( +( +( +( +( +(Z      r2   r   c                       e Zd Zddddddddd	d
ddddef fd	Zd Zd Zej        j	        d             Z
ej        j	        dd            Zej        j	        dd            Zdej        fdZddedee         fdZd ZddefdZd Z xZS )r   r     avgr   ra      r   )@            )r   r5   ra   rI   )rI   ra   r5   r   )       @r   ry   ry   TFr   c                    t                                                       || _        |dv sJ || _        || _        t          |          } t          |          |          } t          |          |          } t          |          |          }t          |          |k    sJ g | _        t          dd||d                   | _	        d t          j        d|t          |                                        |          D             }d}|d         }g }t          |          D ]}|t          |||         ||         |dk    ||         ||         ||         |
|	||||         |          gz  }||         }|||         z  }| xj        t!          |dd|z  z  d	| 
          gz  c_        t#          j        | | _        |d         x| _        | _        t#          j        |          | _        |dk    rt#          j        |d         |          nt#          j                    | _        |                     | j                   d S )Nr    rN   ra   r   r   c                 6    g | ]}|                                 S  )tolist)r   r:   s     r1   r   z7PyramidVisionTransformerV2.__init__.<locals>.<listcomp>;  s     ```aqxxzz```r2   )rO   r   r   r   rP   r\   r   r]   r^   rW   rU   r   r   r5   stages.)num_chs	reductionmoduler`   )r   r   num_classesglobal_pooldepthslenr   feature_infor   patch_embedrr   linspacesumsplitr   r   dictr   
Sequentialstagesnum_featureshead_hidden_sizer(   	head_dropr   r"   headapply_init_weights)r*   r   r   r   r   
embed_dimsrP   	sr_ratios
mlp_ratiosr^   linear	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   
num_stagesdprcurprev_dimr   r   r0   s                         r1   r   z#PyramidVisionTransformerV2.__init__  st   $ 	&k))))&[[
*Yz**:66
)Ij)))44	)Ij)))44	::----, m	
 
 
 a`5>!^S[[#Q#Q#W#WX^#_#_```a=z"" 	d 	dA4"1Qiq5#A,"1$Q-"!((a&%    F "!}H6!9C$x1q!t8Ta^_TaTa"b"b"b!ccmV, 5?rNBD1I..>IAooBIjnk:::SUS^S`S`	

4%&&&&&r2   c                 H   t          |t          j                  rbt          |j        d           t          |t          j                  r.|j        )t          j                            |j        d           d S d S d S t          |t          j                  r|j	        d         |j	        d         z  |j
        z  }||j        z  }|j        j                            dt          j        d|z                       |j        "|j        j                                         d S d S d S )Ng{Gz?)stdr   r   g       @)r   r   r   r   weightr   init	constant_r$   rL   out_channelsr   datanormal_mathsqrtzero_)r*   mfan_outs      r1   r   z(PyramidVisionTransformerV2._init_weights[  s   a## 		$!(,,,,!RY'' -AF,>!!!&!,,,,,- -,>,>29%% 	$mA&q)99ANJG GHM!!!TYsW}%=%=>>>v!!!#####	$ 	$ "!r2   c                     d| j         _        d S )NF)r   requires_gradr*   s    r1   freeze_patch_embz+PyramidVisionTransformerV2.freeze_patch_embg  s    ).&&&r2   c                     i S r   r   r   s    r1   no_weight_decayz*PyramidVisionTransformerV2.no_weight_decayj  s    	r2   c                 (    t          dd          }|S )Nz^patch_embedz^stages\.(\d+))stemr   )r   )r*   coarsematchers      r1   group_matcherz(PyramidVisionTransformerV2.group_matchern  s#     $
 
 
 r2   c                 (    | j         D ]	}||_        
d S r   )r   r   )r*   enabless      r1   set_grad_checkpointingz1PyramidVisionTransformerV2.set_grad_checkpointingv  s(     	* 	*A#)A  	* 	*r2   returnc                     | j         S r   )r   r   s    r1   get_classifierz)PyramidVisionTransformerV2.get_classifier{  s
    yr2   Nr   r   c                     || _         ||dv sJ || _        |dk    rt          j        | j        |          nt          j                    | _        d S )Nr   r   )r   r   r   r   r   r"   r   )r*   r   r   s      r1   reset_classifierz+PyramidVisionTransformerV2.reset_classifier~  sY    &"+----*D>IAooBIdnk:::SUS^S`S`			r2   c                 Z    |                      |          }|                     |          }|S r   )r   r   r   s     r1   forward_featuresz+PyramidVisionTransformerV2.forward_features  s)    QKKNNr2   
pre_logitsc                     | j         r|                    d          }|                     |          }|r|n|                     |          S )N)r`   rc   rd   )r   meanr   r   )r*   r:   r   s      r1   forward_headz'PyramidVisionTransformerV2.forward_head  sJ     	%8$$ANN10qqDIIaLL0r2   c                 Z    |                      |          }|                     |          }|S r   )r   r   r   s     r1   r>   z"PyramidVisionTransformerV2.forward  s-    !!!$$a  r2   F)Tr   )r?   r@   rA   r   r   r   r   rr   rs   ignorer   r   r   r   Moduler   rC   r   strr   r   ru   r   r>   rD   rE   s   @r1   r   r     s        *""' !C' C' C' C' C' C'J
$ 
$ 
$/ / / Y   Y    Y* * * *	    a aC ahsm a a a a  
1 1$ 1 1 1 1      r2   c                    d| v r| S i }ddl }|                                 D ]\  }}|                    d          rX|                    dd          }|                    dd          }|                    dd	          }|                    d
d          }|                    dd          }|                    dd |          }|                    dd |          }|||<   |S )z$ Remap original checkpoints -> timm zpatch_embed.proj.weightr   Nr   patch_embed1patch_embed2zstages.1.downsamplepatch_embed3zstages.2.downsamplepatch_embed4zstages.3.downsamplezdwconv.dwconvr%   zblock(\d+).(\d+)c                 ~    dt          |                     d                    dz
   d|                     d           S )Nr   r   z.blocks.r5   rC   groupr:   s    r1   <lambda>z&checkpoint_filter_fn.<locals>.<lambda>  s<    2eC

OOa<O2e2eYZY`Y`abYcYc2e2e r2   z
^norm(\d+)c                 T    dt          |                     d                    dz
   dS )Nr   r   z.normr  r  s    r1   r  z&checkpoint_filter_fn.<locals>.<lambda>  s'    ,Pc!''!**oo6I,P,P,P r2   )reitems
startswithreplacesub)
state_dictmodelout_dictr  ro   rp   s         r1   checkpoint_filter_fnr    s     J..HIII  "" 	 	1<<&& 	A		.-88A		.*?@@A		.*?@@A		.*?@@AIIox00FF&(e(eghiiFF="P"PRSTTOr2   Fc                     t          t          d                    }|                    d|          }t          t          | |ft
          t          d|          d|}|S )Nra   out_indicesT)flatten_sequentialr  )pretrained_filter_fnfeature_cfg)tupler   popr   r   r  r   )variant
pretrainedkwargsdefault_out_indicesr  r  s         r1   _create_pvt2r%    sp    a//**],?@@K " 2DkJJJ   E Lr2   r   c                 6    | dddddt           t          dddd	|S )
Nr   )r      r'  )rN   rN   g?bicubiczpatch_embed.projr   F)urlr   
input_size	pool_sizecrop_pctinterpolationr   r   
first_conv
classifierfixed_input_sizer   )r)  r#  s     r1   _cfgr1    s8    4}SY)%.B(TY	 
  r2   ztimm/)	hf_hub_id)zpvt_v2_b0.in1kzpvt_v2_b1.in1kzpvt_v2_b2.in1kzpvt_v2_b3.in1kzpvt_v2_b4.in1kzpvt_v2_b5.in1kzpvt_v2_b2_li.in1kr   c           	      X    t          ddd          }t          dd| it          |fi |S )Nr5   r5   r5   r5   )    r      r   r   r5      rI   r   r   rP   	pvt_v2_b0r"  )r:  r   r%  r"  r#  
model_argss      r1   r:  r:    sB    \6HT`aaaJYY
Yd:>X>XQW>X>XYYYr2   c           	      X    t          ddd          }t          dd| it          |fi |S )Nr4  r   r   i@  r   r7  r9  	pvt_v2_b1r"  )r@  r;  r<  s      r1   r@  r@    B    \6IUabbbJYY
Yd:>X>XQW>X>XYYYr2   c           	      X    t          ddd          }t          dd| it          |fi |S )Nr   r?  r7  r9  	pvt_v2_b2r"  )rC  r;  r<  s      r1   rC  rC    rA  r2   c           	      X    t          ddd          }t          dd| it          |fi |S )N)r   ra      r   r?  r7  r9  	pvt_v2_b3r"  )rF  r;  r<  s      r1   rF  rF    B    ]7JVbcccJYY
Yd:>X>XQW>X>XYYYr2   c           	      X    t          ddd          }t          dd| it          |fi |S )N)r   rI      r   r?  r7  r9  	pvt_v2_b4r"  )rJ  r;  r<  s      r1   rJ  rJ    rG  r2   c           	      Z    t          dddd          }t          dd| it          |fi |S )	N)r   r   (   r   r?  r7  )ra   ra   ra   ra   )r   r   rP   r   	pvt_v2_b5r"  )rM  r;  r<  s      r1   rM  rM    sN    )<amo o oJYY
Yd:>X>XQW>X>XYYYr2   c           	      Z    t          dddd          }t          dd| it          |fi |S )	Nr   r?  r7  T)r   r   rP   r   pvt_v2_b2_lir"  )rO  r;  r<  s      r1   rO  rO    sN    (;|\`b b bJ\\:\jA[A[TZA[A[\\\r2   r  )r   )2r   r   typingr   r   r   r   rr   torch.nnr   torch.nn.functional
functionalrh   torch.utils.checkpointutilsr   	timm.datar   r	   timm.layersr
   r   r   r   r   r   _builderr   	_registryr   r   __all__r  r   rG   rx   r   r   r   r  r%  r1  default_cfgsr:  r@  rC  rF  rJ  rM  rO  r   r2   r1   <module>r\     s\   "  2 2 2 2 2 2 2 2 2 2 2 2                 + + + + + + + + + A A A A A A A A ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` ` * * * * * * < < < < < < < <'
(    29   DL L L L L	 L L L^+ + + + +BI + + +\    	   (= = = = =BI = = =@~ ~ ~ ~ ~ ~ ~ ~B  (       %$dW---dW---dW---dW---dW---dW---000& &   Z Z-G Z Z Z Z
 Z Z-G Z Z Z Z
 Z Z-G Z Z Z Z
 Z Z-G Z Z Z Z
 Z Z-G Z Z Z Z
 Z Z-G Z Z Z Z ] ]0J ] ] ] ] ] ]r2   