
    NgWJ                        d Z ddlZddlmZ ddlmZmZ ddlmZm	Z	m
Z
mZmZmZmZ ddlmZ ddlmZ ddlmZmZ d	gZ G d
 dej                  Z G d dej                  Z G d dej                  Z G d d	ej                  ZddZddZ e ed           ed          d          Zeddefd            Zeddefd            Z dS )z Visformer

Paper: Visformer: The Vision-friendly Transformer - https://arxiv.org/abs/2104.12533

From original at https://github.com/danczs/Visformer

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    NIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)	to_2tupletrunc_normal_DropPath
PatchEmbedLayerNorm2dcreate_classifieruse_fused_attn   )build_model_with_cfg)checkpoint_seq)register_modelgenerate_default_cfgs	Visformerc                   <     e Zd Zddej        dddf fd	Zd Z xZS )
SpatialMlpN           Fc           	         t                                                       |p|}|p|}t          |          }|| _        || _        || _        | j        r|dk     r	|dz  dz  }n|dz  }|| _        || _        t          j	        ||dddd          | _
         |            | _        t          j        |d                   | _        | j        r5t          j	        ||ddd| j        d	          | _         |            | _        nd | _        d | _        t          j	        ||dddd          | _        t          j        |d                   | _        d S )
N         r   r   Fstridepaddingbias   )r   r   groupsr   )super__init__r   in_featuresout_featuresspatial_convhidden_featuresgroupnnConv2dconv1act1Dropoutdrop1conv2act2conv3drop3)
selfr#   r&   r$   	act_layerdropr'   r%   
drop_probs	__class__s
            Q/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/visformer.pyr"   zSpatialMlp.__init__   s`    	#2{)8[t__
&(( 	2qyy"-/Q"6"-/.
Y{OQqRSZ_```
IKK	Z
1..
 	!AqQUQ[bgi i iDJ!	DIIDJDIYaST[`aaa
Z
1..


    c                 :   |                      |          }|                     |          }|                     |          }| j        *|                     |          }|                     |          }|                     |          }|                     |          }|S N)r*   r+   r-   r.   r/   r0   r1   r2   xs     r7   forwardzSpatialMlp.forward=   sx    JJqMMIIaLLJJqMM:!

1A		!AJJqMMJJqMMr8   )__name__
__module____qualname__r(   GELUr"   r=   __classcell__r6   s   @r7   r   r      sc         !g$/ $/ $/ $/ $/ $/L	 	 	 	 	 	 	r8   r   c                   R     e Zd ZU ej        j        e         ed<   d fd	Zd Z	 xZ
S )	Attention
fused_attnr         ?r   c                    t                                                       || _        || _        t	          ||z  |z            }|| _        |dz  | _        t          d          | _        t          j
        |||z  dz  dddd          | _        t          j        |          | _        t          j
        | j        | j        z  |dddd          | _        t          j        |          | _        d S )	Ng      T)experimentalr   r   r   Fr   )r!   r"   dim	num_headsroundhead_dimscaler   rF   r(   r)   qkvr,   	attn_dropproj	proj_drop)r2   rJ   rK   head_dim_ratiorP   rR   rM   r6   s          r7   r"   zAttention.__init__L   s    "	)N:;; %
(d;;;9S(Y"6":AaQRY^___I..Idmdn<c1QXY`efff	I..r8   c                 .   |j         \  }}}}|                     |                              |d| j        | j        d                              ddddd          }|                    d          \  }}}| j        rrt          j	        j
                            |                                |                                |                                | j        r| j        j        nd          }nQ||                    d	d          z  | j        z  }	|	                    d
          }	|                     |	          }	|	|z  }|                    dddd                              |d||          }|                     |          }|                     |          }|S )Nr   r   r   r      r   )	dropout_p)rJ   )shaperO   reshaperK   rM   permuteunbindrF   torchr(   
functionalscaled_dot_product_attention
contiguoustrainingrP   p	transposerN   softmaxrQ   rR   )
r2   r<   BCHWqkvattns
             r7   r=   zAttention.forwardZ   sg   W
1aHHQKK1dndmRHHPPQRTUWXZ[]^__((1++1a? 		#@@.2mC$.** A  AA
 B+++tz9D<<B<''D>>$''DqAIIaAq!!))!RA66IIaLLNN1r8   )r   rG   r   r   )r>   r?   r@   r]   jitFinalbool__annotations__r"   r=   rB   rC   s   @r7   rE   rE   I   sc         	%%%%/ / / / / /      r8   rE   c            
       D     e Zd Zdddddej        edddf
 fd	Zd Z xZS )BlockrG         @r   r   Fc                    t                                                       || _        |dk    rt          |          nt	          j                    | _        |rd | _        d | _        n) |	|          | _        t          |||||          | _         |	|          | _
        t          |t          ||z            |||
|          | _        d S )Nr   )rK   rS   rP   rR   )r#   r&   r3   r4   r'   r%   )r!   r"   r%   r   r(   Identity	drop_pathnorm1rl   rE   norm2r   intmlp)r2   rJ   rK   rS   	mlp_ratiorR   rP   rv   r3   
norm_layerr'   attn_disabledr%   r6   s                r7   r"   zBlock.__init__q   s     	(09B),,,BKMM 	DJDII#CDJ!#-##  DI  Z__
i00%
 
 
r8   c                    | j         >||                     |                      |                     |                              z   }||                     |                     |                     |                              z   }|S r:   )rl   rv   rw   rz   rx   r;   s     r7   r=   zBlock.forward   sg    9 DNN499TZZ]]#;#;<<<Atxx

166777r8   )	r>   r?   r@   r(   rA   r
   r"   r=   rB   rC   s   @r7   rr   rr   p   so        
 g"'
 '
 '
 '
 '
 '
R      r8   rr   c                       e Zd Zddddddddd	d
d
d
d
d
eddddddddf fd	Zd Zej        j        dd            Z	ej        j        dd            Z
ej        j        dej        fd            Zd dedefdZd ZddefdZd Z xZS )!r         r              r   rs   r   111TFr   avgNc                 ^
  	 t                                                       t          |          }|| _        | _        || _        || _        || _        || _        t          |t          t          f          r%|\  | _        | _        | _        t          |          }n(|dz  x| _        | _        || j        z
  | j        z
  | _        || _        d| _        d t%          j        d||          D             | j        r0d | _        t+          |||d          | _        fd|D             }n| j        6d | _        t+          |dz  |dz  |d          | _        fd|D             }nt/          j        t/          j        || j        d	ddd
          t/          j        | j                  t/          j        d                    | _        d |D             }t+          |dz  | j        dz  |d          | _        fd|D             }| j        rz| j        r+t/          j        t%          j        dg|R            | _        n-t/          j        t%          j        ddz  g|R            | _        t/          j        |          | _         nd | _        t/          j        	f
dtC          | j                  D              | _"        | j        sht+          |dz  dz  |d          | _#        fd|D             }| j        r+t/          j        t%          j        dg|R            | _$        nd | _$        nd | _#        t/          j        	f
dtC          | j        | j        | j        z             D              | _%        | j        skt+          |dz  dz  |d          | _&        fd|D             }| j        r.t/          j        t%          j        ddz  g|R            | _'        nd | _'        nd | _&        t/          j        	f
dtC          | j        | j        z   |          D              | _(        | j        rndz  x| _)        | _*         | j)                  | _+        tY          | j)        | j        |          \  }}|| _-        t/          j        |
          | _.        || _/        | j        rIta          | j        d           | j        s,ta          | j$        d           ta          | j'        d           | 1                    | j2                   d S )Nr   Fc                 6    g | ]}|                                 S  )item.0r<   s     r7   
<listcomp>z&Visformer.__init__.<locals>.<listcomp>   s     JJJAqvvxxJJJr8   r   )img_size
patch_sizein_chans	embed_dimr|   flattenc                     g | ]}|z  S r   r   r   r<   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>   s    :::AZ:::r8   r   c                      g | ]
}|d z  z  S r   r   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>   "    EEEqA*/2EEEr8      r   T)inplacec                     g | ]}|d z  S r   r   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>   s    555qAF555r8   rV   c                      g | ]
}|d z  z  S )rV   r   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>   r   r8   r   )rb   c                 ~   
 g | ]9}t          d z  	d
|         d         dk    d         dk              :S )r   g      ?r   01rJ   rK   rS   r{   rR   rP   rv   r|   r'   r}   r%   rr   r   iattn_drop_rate
attn_stagedprr   r'   r{   r|   rK   proj_drop_rater%   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>   su     &
 &
 &
  qL#"#((a&%)!}3*1o4  &
 &
 &
r8   r   c                      g | ]
}|d z  z  S r   r   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>  "    AAA1jAo.AAAr8   c                 x   
 g | ]6}t          	d 
|         d         dk    d         dk              7S )rG   r   r   r   r   r   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>"  sq     &
 &
 &
  #"#((a&%)!}3*1o4  &
 &
 &
r8   c                      g | ]
}|d z  z  S r   r   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>=  r   r8   c                 ~   
 g | ]9}t          d z  	d
|         d          dk    d          dk              :S )r   rG   r   r   r   r   r   s     r7   r   z&Visformer.__init__.<locals>.<listcomp>D  su     &
 &
 &
  M#"#((a&%)!}3*1o4  &
 &
 &
r8   	pool_type{Gz?std)3r!   r"   r   num_classesr   init_channelsr   vit_stem	conv_init
isinstancelisttuple
stage_num1
stage_num2
stage_num3sumuse_pos_embedgrad_checkpointingr]   linspacestemr	   patch_embed1r(   
Sequentialr)   BatchNorm2dReLU	Parameterzeros
pos_embed1r,   pos_droprangestage1patch_embed2
pos_embed2stage2patch_embed3
pos_embed3stage3num_featureshead_hidden_sizenormr   global_pool	head_dropheadr   apply_init_weights)r2   r   r   r   r   r   r   depthrK   r{   	drop_ratepos_drop_rater   r   drop_path_rater|   r   r   r%   r   r'   r   r   
embed_normr   r   r6   s     `   ` ``  `` `` ` `    @r7   r"   zVisformer.__init__   sb   4 	X&&&"*  "edE]++ 	H@E=DOT_doJJEE05
:DOdo#do5GDO*"'JJ>5!I!IJJJ= &	FDI *!%!#%! ! !D ;::::::HH!) 	$.%)Q%'1n)!% % %! FEEEHEEEMIh(:AaQRY^___N4#566GD))) 	
 65H555$.%)Q!/'1n)!% % %! FEEEHEEE 	#} X"$,u{1i/S(/S/S/S"T"T"$,u{1il/VX/V/V/V"W"WJ777DMM"DOm &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 4?++&
 &
 &
 $ } 	% *!%?"a#%! ! !D BAAAAAAH! '"$,u{1i/S(/S/S/S"T"T"& $Dm &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 4?DODO,KLL&
 &
 &
 $ } 	% *!%?"#a-%! ! !D BAAAAAAH! '"$,u{1ik/UH/U/U/U"V"V"& $Dm &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 &
 4?4?:EBB&
 &
 &
 " BF4aIIT]`aTaaD1Jt011	 .d.?AQ]hiiiT&I..	  	9$/t4444= 9do48888do48888

4%&&&&&r8   c                    t          |t          j                  rFt          |j        d           |j        't          j                            |j        d           d S d S t          |t          j                  rs| j	        r(t          j        
                    |j        dd           nt          |j        d           |j        )t          j                            |j        d           d S d S d S )Nr   r   r   fan_outrelu)modenonlinearityr   )r   r(   Linearr   weightr   init	constant_r)   r   kaiming_normal_)r2   ms     r7   r   zVisformer._init_weightsf  s    a## 
	.!(----v!!!!&!,,,,, "!29%% 	.~ 2''yv'VVVVahD1111v!!!!&"-----	. 	.
 "!r8   c                 6    t          d|rdndd fddg          S )Nz^patch_embed1|pos_embed1|stemz^stage(\d+)\.(\d+))z^(?:patch_embed|pos_embed)(\d+))r   )z^norm)i )r   blocks)dict)r2   coarses     r7   group_matcherzVisformer.group_matchers  s;    1*0K&&6KTR:$
 
 
 	
r8   c                     || _         d S r:   )r   )r2   enables     r7   set_grad_checkpointingz Visformer.set_grad_checkpointing~  s    "(r8   returnc                     | j         S r:   )r   )r2   s    r7   get_classifierzVisformer.get_classifier  s
    yr8   r   r   c                 f    || _         t          | j        | j         |          \  | _        | _        d S )Nr   )r   r   r   r   r   )r2   r   r   s      r7   reset_classifierzVisformer.reset_classifier  s3    &&78I4K[gr&s&s&s#$)))r8   c                    | j         |                      |          }|                     |          }| j        |                     || j        z             }| j        r4t
          j                                        st          | j	        |          }n| 	                    |          }| j
        9| 
                    |          }| j        |                     || j        z             }| j        r4t
          j                                        st          | j        |          }n|                     |          }| j        9|                     |          }| j        |                     || j        z             }| j        r4t
          j                                        st          | j        |          }n|                     |          }|                     |          }|S r:   )r   r   r   r   r   r]   rm   is_scriptingr   r   r   r   r   r   r   r   r   r;   s     r7   forward_featureszVisformer.forward_features  s   9 		!A a  ?&a$/122A" 	59+A+A+C+C 	t{A..AAAA (!!!$$A*MM!do"566" 	59+A+A+C+C 	t{A..AAAA (!!!$$A*MM!do"566" 	59+A+A+C+C 	t{A..AAAAIIaLLr8   
pre_logitsc                     |                      |          }|                     |          }|r|n|                     |          S r:   )r   r   r   )r2   r<   r   s      r7   forward_headzVisformer.forward_head  s?    QNN10qqDIIaLL0r8   c                 Z    |                      |          }|                     |          }|S r:   )r   r   r;   s     r7   r=   zVisformer.forward  s-    !!!$$a  r8   F)T)r   )r>   r?   r@   r
   r"   r   r]   rm   ignorer   r   r(   Moduler   ry   strr   r   ro   r   r=   rB   rC   s   @r7   r   r      s        "1B' B' B' B' B' B'H. . . Y
 
 
 
 Y) ) ) ) Y	    t tC tc t t t t" " "H1 1$ 1 1 1 1
      r8   Fc                 x    |                     dd           rt          d          t          t          | |fi |}|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretraineddefault_cfgkwargsmodels        r7   _create_visformerr    sF    zz/4(( [YZZZ GZJJ6JJELr8    c                 6    | ddddddt           t          ddd	|S )
Nr   )r   r   r   )r   r   g?bicubicTzstem.0r   )urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizemeanr   
first_conv
classifierr   )r  r	  s     r7   _cfgr    s7    =v%.Bf   r8   ztimm/)	hf_hub_id)zvisformer_tiny.in1kzvisformer_small.in1kr   c                     t          ddddddddt          j        d	t          j        
          }t          dd| it          |fi |}|S )Nr      r   rV   rV   r   rs   r   011100Tr   r   r   rK   r{   r'   r   r%   r|   r   r   visformer_tinyr  )r   r   r(   r   r  r  r	  	model_cfgr
  s       r7   r   r     sf    CyAQS[\uSW># # #I cc:ciIbIb[aIbIbccELr8   c                     t          ddddddddt          j        d	t          j        
          }t          dd| it          |fi |}|S )Nr   r   r  r   rs   r   r  r  Tr  visformer_smallr  )r%  r!  r"  s       r7   r%  r%    sf    CyAQS[\uSW># # #I ddJd$yJcJc\bJcJcddELr8   )FN)r  r   )!__doc__r]   torch.nnr(   	timm.datar   r   timm.layersr   r   r   r	   r
   r   r   _builderr   _manipulater   	_registryr   r   __all__r   r   rE   rr   r   r  r  default_cfgsr   r%  r   r8   r7   <module>r/     sA           A A A A A A A A v v v v v v v v v v v v v v v v v v * * * * * * ' ' ' ' ' ' < < < < < < < <-0 0 0 0 0 0 0 0f$ $ $ $ $	 $ $ $N. . . . .BI . . .bU U U U U	 U U Up       %$4'222 D7333& &    )      9      r8   