
    %hB                        d dl Z d dlmZ d dlZd dlmZ d dlmZmZm	Z	 d dl
mZ  e       Z
 G d dej                        Z G d dej                        Z G d	 d
ej                        Z G d dej                        Z G d dej                        Z G d dej                        ZddZ G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Zy)    N)partial)DropPath	to_2tupletrunc_normal_)Configc                   H     e Zd Zddej                  df fd	Zd Zd Z xZS )MlpN        c                 H   t         |           |xs |}|xs |}t        j                  ||      | _        t        |      | _         |       | _        t        j                  ||      | _        t        j                  |      | _
        | j                  | j                         y N)super__init__nnLinearfc1DWConvdwconvactfc2Dropoutdropapply_init_weights)selfin_featureshidden_featuresout_features	act_layerr   	__class__s         9/var/www/html/mariraj/BiRefNet/models/backbones/pvt_v2.pyr   zMlp.__init__   s}    #2{)8[99[/:_-;99_l;JJt$	

4%%&    c                 p   t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y t        |t        j                        r|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                   d|z               |j
                  %|j
                  j                  j#                          y y y Ng{Gz?)stdr   g      ?   g       @
isinstancer   r   r   weightbiasinit	constant_	LayerNormConv2dkernel_sizeout_channelsgroupsdatanormal_mathsqrtzero_r   mfan_outs      r    r   zMlp._init_weights   !   a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh,299%mmA&q)99ANNJG GHHMM!!!TYYsW}%=>vv!!!# "	 &r!   c                     | j                  |      }| j                  |||      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|S r   )r   r   r   r   r   r   xHWs       r    forwardzMlp.forward)   sZ    HHQKKK1a HHQKIIaLHHQKIIaLr!   )	__name__
__module____qualname__r   GELUr   r   r?   __classcell__r   s   @r    r	   r	      s#    48tWYW^W^eg 
'$r!   r	   c                   ,     e Zd Zd fd	Zd Zd Z xZS )	Attentionc                 t   t         	|           ||z  dk(  sJ d| d| d       || _        || _        ||z  }|xs |dz  | _        t        j                  |||      | _        t        j                  ||dz  |      | _        || _	        t        j                  |      | _        t        j                  ||      | _        t        j                  |      | _        || _        |dkD  r8t        j                  ||||	      | _        t        j"                  |      | _        | j'                  | j(                         y )
Nr   zdim z  should be divided by num_heads .g      )r)      r%   )r.   stride)r   r   dim	num_headsscaler   r   qkvattn_drop_probr   	attn_dropproj	proj_dropsr_ratior-   srr,   normr   r   )
r   rL   rM   qkv_biasqk_scalerR   rT   rU   head_dimr   s
            r    r   zAttention.__init__4   s   Y!#]tC50PQZP[[\%]]#")#1T!1
3(3))Cqx8'I.IIc3'	I. a<iiShxPDGS)DI

4%%&r!   c                 p   t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y t        |t        j                        r|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                   d|z               |j
                  %|j
                  j                  j#                          y y y r#   r&   r6   s      r    r   zAttention._init_weightsK   r9   r!   c                    |j                   \  }}}| j                  |      j                  ||| j                  || j                  z        j	                  dddd      }| j
                  dkD  r|j	                  ddd      j                  ||||      }| j                  |      j                  ||d      j	                  ddd      }| j                  |      }| j                  |      j                  |dd| j                  || j                  z        j	                  ddddd      }	nN| j                  |      j                  |dd| j                  || j                  z        j	                  ddddd      }	|	d   |	d   }}
t        j                  r[t        j                  j                  j                  ||
|d | j                  d      j!                  dd      j                  |||      }nk||
j!                  d	d      z  | j"                  z  }|j%                  d
      }| j'                  |      }||z  j!                  dd      j                  |||      }| j)                  |      }| j+                  |      }|S )Nr   rJ   r%         F)	attn_mask	dropout_p	is_causal)rL   )shaperO   reshaperM   permuterU   rV   rW   rP   configSDPA_enabledtorchr   
functionalscaled_dot_product_attentionrQ   	transposerN   softmaxrR   rS   rT   )r   r<   r=   r>   BNCrO   x_rP   kvattns                r    r?   zAttention.forwardZ   s0   ''1aFF1IaDNNA4GHPPQRTUWXZ[\==11a#++Aq!Q7B$$Q2.66q!Q?B2B$$QAt~~qDNN?RS[[\]_`bcefhijB##Ar1dnna4>>>QRZZ[\^_abdeghiB!ube1##@@1a$*=*= A  i1oggaA. 
 B++tzz9D<<B<'D>>$'D$$Q*221a;AIIaLNN1r!   )   FNr
   r
   r%   )r@   rA   rB   r   r   r?   rD   rE   s   @r    rG   rG   3   s    '.$r!   rG   c            	       f     e Zd Zddddddej                  ej
                  df	 fd	Zd Zd Z xZ	S )	Blockg      @FNr
   r%   c           	      P   t         |            |
|      | _        t        |||||||      | _        |dkD  rt        |      nt        j                         | _         |
|      | _	        t        ||z        }t        |||	|      | _        | j                  | j                         y )N)rM   rX   rY   rR   rT   rU   r
   )r   r   r   r   )r   r   norm1rG   rt   r   r   Identity	drop_pathnorm2intr	   mlpr   r   )r   rL   rM   	mlp_ratiorX   rY   r   rR   r{   r   
norm_layerrU   mlp_hidden_dimr   s                r    r   zBlock.__init__z   s    _
(X4(D	
 1:B),BKKM_
S9_-3R[bfg

4%%&r!   c                 p   t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y t        |t        j                        r|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                   d|z               |j
                  %|j
                  j                  j#                          y y y r#   r&   r6   s      r    r   zBlock._init_weights   r9   r!   c                     || j                  | j                  | j                  |      ||            z   }|| j                  | j                  | j	                  |      ||            z   }|S r   )r{   rt   ry   r~   r|   r;   s       r    r?   zBlock.forward   sW    tyyA1=>>txx

1q!<==r!   )
r@   rA   rB   r   rC   r,   r   r   r?   rD   rE   s   @r    rw   rw   x   s1    13edY[giR\\TU' $r!   rw   c                   0     e Zd ZdZd fd	Zd Zd Z xZS )OverlapPatchEmbedz Image to Patch Embedding
    c           	         t         |           t        |      }t        |      }|| _        || _        |d   |d   z  |d   |d   z  c| _        | _        | j
                  | j                  z  | _        t        j                  |||||d   dz  |d   dz  f      | _
        t        j                  |      | _        | j                  | j                         y )Nr   r%   rJ   )r.   rK   padding)r   r   r   img_size
patch_sizer=   r>   num_patchesr   r-   rS   r,   rW   r   r   )r   r   r   rK   in_channels	embed_dimr   s         r    r   zOverlapPatchEmbed.__init__   s    X&z*
 $!!
15x{jQRm7S66DFF?IIk9*U['1!}'9:a=A;M&NP	LL+	

4%%&r!   c                 p   t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y t        |t        j                        r|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                   d|z               |j
                  %|j
                  j                  j#                          y y y r#   r&   r6   s      r    r   zOverlapPatchEmbed._init_weights   r9   r!   c                     | j                  |      }|j                  \  }}}}|j                  d      j                  dd      }| j	                  |      }|||fS )NrJ   r%   )rS   rd   flattenrl   rW   )r   r<   _r=   r>   s        r    r?   zOverlapPatchEmbed.forward   sR    IIaLWW
1aIIaL""1a(IIaL!Qwr!   )      r_   r]      )r@   rA   rB   __doc__r   r   r?   rD   rE   s   @r    r   r      s    '$r!   r   c                        e Zd Zddddg dg dg ddd	d
d
d
ej                  g dg df fd	Zd ZddZd Zd Z	e
j                  j                  d        Zd ZddZd Zd Z xZS )PyramidVisionTransformerImprr      r]   i  )@            )r%   rJ   r_   ru   r_   r_   r_   r_   FNr
   r]   r_      r]   ru   r_   rJ   r%   c                 |   t         |           || _        || _        t	        |dd||d         | _        t	        |dz  dd|d   |d         | _        t	        |dz  dd|d   |d         | _        t	        |d	z  dd|d   |d         | _        t        j                  d|t        |            D cg c]  }|j                          }}d}t        j                  t        |d         D cg c])  }t!        |d   |d   |d   ||	|
||||z      ||d   

      + c}      | _         ||d         | _        ||d   z  }t        j                  t        |d         D cg c])  }t!        |d   |d   |d   ||	|
||||z      ||d   

      + c}      | _         ||d         | _        ||d   z  }t        j                  t        |d         D cg c])  }t!        |d   |d   |d   ||	|
||||z      ||d   

      + c}      | _         ||d         | _        ||d   z  }t        j                  t        |d         D cg c])  }t!        |d   |d   |d   ||	|
||||z      ||d   

      + c}      | _         ||d         | _        | j3                  | j4                         y c c}w c c}w c c}w c c}w c c}w )Nr   r_   r   )r   r   rK   r   r   r]   rJ   r%   ru   r   )
rL   rM   r   rX   rY   r   rR   r{   r   rU   )r   r   num_classesdepthsr   patch_embed1patch_embed2patch_embed3patch_embed4ri   linspacesumitemr   
ModuleListrangerw   block1ry   block2r|   block3norm3block4norm4r   r   )r   r   r   r   r   
embed_dimsrM   
mlp_ratiosrX   rY   	drop_rateattn_drop_ratedrop_path_rater   r   	sr_ratiosr<   dprcurir   s                       r    r   z%PyramidVisionTransformerImpr.__init__   s8    	& .xAVWep8B1G-x1}QR[\jtuvjw8B1G-x1}QR[\jtuvjw8B1G-x2~RS\]kuvwkx8B1G "'>3v;!OPAqvvxPPmm 6!9%	%' 	 &+11AYaltnC!GYcq\&# %' (
  
1.
vaymm 6!9%	%' 	 &+11AYaltnC!GYcq\&# %' (
  
1.
vaymm 6!9%	%' 	 &+11AYaltnC!GYcq\&# %' (
  
1.
vaymm 6!9%	%' 	 &+11AYaltnC!GYcq\&# %' (
  
1.

 	

4%%&I Q%'%'%'%'s   2J%,.J*.J/.J4?.J9c                 p   t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y t        |t        j                        r|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                   d|z               |j
                  %|j
                  j                  j#                          y y y r#   r&   r6   s      r    r   z*PyramidVisionTransformerImpr._init_weights  r9   r!   c                 *    t        |t              rd}y y )Nr%   )r'   str)r   
pretrainedloggers      r    init_weightsz)PyramidVisionTransformerImpr.init_weights  s    j#&F 'r!   c                    t        j                  d|t        | j                              D cg c]  }|j	                          }}d}t        | j                  d         D ]&  }|||z      | j                  |   j                  _        ( || j                  d   z  }t        | j                  d         D ]&  }|||z      | j                  |   j                  _        ( || j                  d   z  }t        | j                  d         D ]&  }|||z      | j                  |   j                  _        ( || j                  d   z  }t        | j                  d         D ]&  }|||z      | j                  |   j                  _        ( y c c}w )Nr   r%   rJ   r]   )ri   r   r   r   r   r   r   r{   	drop_probr   r   r   )r   r   r<   r   r   r   s         r    reset_drop_pathz,PyramidVisionTransformerImpr.reset_drop_path  s`   !&>3t{{CS!TUAqvvxUUt{{1~& 	>A14S1WDKKN$$.	> 	t{{1~t{{1~& 	>A14S1WDKKN$$.	> 	t{{1~t{{1~& 	>A14S1WDKKN$$.	> 	t{{1~t{{1~& 	>A14S1WDKKN$$.	> Vs   Fc                 &    d| j                   _        y )NF)r   requires_gradr   s    r    freeze_patch_embz-PyramidVisionTransformerImpr.freeze_patch_emb+  s    */'r!   c                 
    h dS )N>   	cls_token
pos_embed1
pos_embed2
pos_embed3
pos_embed4 r   s    r    no_weight_decayz,PyramidVisionTransformerImpr.no_weight_decay.  s    TTr!   c                     | j                   S r   )headr   s    r    get_classifierz+PyramidVisionTransformerImpr.get_classifier2  s    yyr!   c                     || _         |dkD  r&t        j                  | j                  |      | _        y t        j                         | _        y )Nr   )r   r   r   r   rz   r   )r   r   global_pools      r    reset_classifierz-PyramidVisionTransformerImpr.reset_classifier5  s6    &>IAoBIIdnnk:	SUS^S^S`	r!   c                    |j                   d   }g }| j                  |      \  }}}t        | j                        D ]  \  }} ||||      } | j	                  |      }|j                  |||d      j                  dddd      j                         }|j                  |       | j                  |      \  }}}t        | j                        D ]  \  }} ||||      } | j                  |      }|j                  |||d      j                  dddd      j                         }|j                  |       | j                  |      \  }}}t        | j                        D ]  \  }} ||||      } | j                  |      }|j                  |||d      j                  dddd      j                         }|j                  |       | j                  |      \  }}}t        | j                         D ]  \  }} ||||      } | j#                  |      }|j                  |||d      j                  dddd      j                         }|j                  |       |S )Nr   r^   r]   r%   rJ   )rd   r   	enumerater   ry   re   rf   
contiguousappendr   r   r|   r   r   r   r   r   r   )r   r<   rn   outsr=   r>   r   blks           r    forward_featuresz-PyramidVisionTransformerImpr.forward_features9  s5   GGAJ ##A&1a, 	FAsAq!A	JJqMIIaAr"**1aA6AACA ##A&1a, 	FAsAq!A	JJqMIIaAr"**1aA6AACA ##A&1a, 	FAsAq!A	JJqMIIaAr"**1aA6AACA ##A&1a, 	FAsAq!A	JJqMIIaAr"**1aA6AACAr!   c                 (    | j                  |      }|S r   )r   )r   r<   s     r    r?   z$PyramidVisionTransformerImpr.forwarda  s    !!!$ r!   r   ) )r@   rA   rB   r   r,   r   r   r   r   r   ri   jitignorer   r   r   r   r?   rD   rE   s   @r    r   r      s|     #t`s'L5[_km "2",,$7'r$
>$0 YYU Ua$Pr!   r   c                   &     e Zd Zd fd	Zd Z xZS )r   c           	      l    t         t        |           t        j                  ||dddd|      | _        y )Nr]   r%   T)r)   r0   )r   r   r   r   r-   r   )r   rL   r   s     r    r   zDWConv.__init__i  s,    fd$&iiS!QSIr!   c                     |j                   \  }}}|j                  dd      j                  ||||      j                         }| j	                  |      }|j                  d      j                  dd      }|S )Nr%   rJ   )rd   rl   viewr   r   r   )r   r<   r=   r>   rn   ro   rp   s          r    r?   zDWConv.forwardm  sg    ''1aKK1""1aA.99;KKNIIaL""1a(r!   )r   )r@   rA   rB   r   r?   rD   rE   s   @r    r   r   h  s    Jr!   r   c                     i }| j                         D ]0  \  }}d|v r"|j                  |j                  d   d||f      }|||<   2 |S )zJ convert patch embedding weight from manual patchify + linear proj to convzpatch_embed.proj.weightr   r]   )itemsre   rd   )
state_dictr   out_dictrr   rs   s        r    _conv_filterr   v  sZ    H  " 1$)		1771:q*jABA
 Or!   c                        e Zd Z fdZ xZS )	pvt_v2_b0c                     t         t        |   dg dg dg ddt        t        j
                  d      g dg d	d
d
       y )Nr_   )    r      r   r%   rJ      ru   ru   ru   r_   r_   Tư>epsrJ   rJ   rJ   rJ   r   r
   皙?
r   r   rM   r   rX   r   r   r   r   r   )r   r   r   r   r   r,   r   kwargsr   s     r    r   zpvt_v2_b0.__init__  s<    i'%7<\hgbll&Elfr# 	( 	/r!   r@   rA   rB   r   rD   rE   s   @r    r   r         / /r!   r   c                        e Zd Z fdZ xZS )	pvt_v2_b1c                     t         t        |   dg dg dg ddt        t        j
                  d      g dg d	d
d
       y )Nr_   r   r   i@  r   r   r   Tr   r   r   r   r
   r   r   )r   r   r   r   r   r,   r   s     r    r   zpvt_v2_b1.__init__  s<    i'%8L]igbll&Elfr# 	( 	/r!   r   rE   s   @r    r   r     r   r!   r   c                         e Zd Zd fd	Z xZS )	pvt_v2_b2c                     t         t        |   dg dg dg ddt        t        j
                  d      g dg d	d
d|       y )Nr_   r   r   r   Tr   r   r   r   r
   r   )r   r   rM   r   rX   r   r   r   r   r   r   )r   r  r   r   r   r,   )r   r   r   r   s      r    r   zpvt_v2_b2.__init__  s?    i'%8L]igbll&Elfr#; 	( 	Hr!   )r]   r   rE   s   @r    r  r    s    H Hr!   r  c                        e Zd Z fdZ xZS )	pvt_v2_b3c                     t         t        |   dg dg dg ddt        t        j
                  d      g dg d	d
d
       y )Nr_   r   r   r   Tr   r   )r]   r_      r]   r   r
   r   r   )r   r  r   r   r   r,   r   s     r    r   zpvt_v2_b3.__init__  <    i'%8L]igbll&Emgs# 	( 	/r!   r   rE   s   @r    r  r    r   r!   r  c                        e Zd Z fdZ xZS )	pvt_v2_b4c                     t         t        |   dg dg dg ddt        t        j
                  d      g dg d	d
d
       y )Nr_   r   r   r   Tr   r   )r]   ru      r]   r   r
   r   r   )r   r	  r   r   r   r,   r   s     r    r   zpvt_v2_b4.__init__  r  r!   r   rE   s   @r    r	  r	    r   r!   r	  c                        e Zd Z fdZ xZS )	pvt_v2_b5c                     t         t        |   dg dg dg ddt        t        j
                  d      g dg d	d
d
       y )Nr_   r   r   r   Tr   r   )r]   r   (   r]   r   r
   r   r   )r   r  r   r   r   r,   r   s     r    r   zpvt_v2_b5.__init__  r  r!   r   rE   s   @r    r  r    r   r!   r  )r   )r3   	functoolsr   ri   torch.nnr   timm.layersr   r   r   rg   r   Moduler	   rG   rw   r   r   r   r   r   r   r  r  r	  r  r   r!   r    <module>r     s        : : 	#")) #LB		 BJ%BII %P(		 (VZ299 ZzRYY /, //, /H, H/, //, //, /r!   