
    NgR                        d Z ddlmZ ddlmZ ddlmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZm Z  dgZ! G d dej"                  Z# G d dej"                  Z$ G d dej"                  Z% G d dej&                  Z' G d dej&                  Z( G d dej"                  Z) G d dej"                  Z* G d dej"                  Z+ G d dej"                  Z, G d  d!ej"                  Z- G d" d#ej"                  Z. G d$ d%ej"                  Z/ G d& dej"                  Z0 G d' d(e0          Z1d) Z2 e3 e3d*d+d,d-.           e3d*d+d/d0.           e3d1d2d3d0.           e3d4d2d,d0.           e3d5d2d6d0.           e3d5d2d6d0d7d89           e3d:d;d<d0d7d89           e3d=d;d>d0d7?           e3d4d2d,d@d7?           e3dAd;dBd@d7?          C
  
        Z4dvdFZ5dwdHZ6 ei dI e6dJK          dL e6dJK          dM e6dJK          dN e6dJK          dO e6dJK          dP e6dJdQR          dS e6dJdQR          dT e6dJdQR          dU e6dJdQR          dV e6dJdQR          dW e6dXY          dZ e6dXY          d[ e6dXY          d\ e6dXY          d] e6dXY          d^ e6dXY          d_ e6dXY           e6dXY           e6dXY           e6dXY          d`          Z7e dxdae0fdb            Z8e dxdae0fdc            Z9e dxdae0fdd            Z:e dxdae0fde            Z;e dxdae0fdf            Z<e dxdae0fdg            Z=e dxdae0fdh            Z>e dxdae0fdi            Z?e dxdae0fdj            Z@e dxdae0fdk            ZAe dxdae0fdl            ZBe dxdae0fdm            ZCe dxdae0fdn            ZDe dxdae0fdo            ZEe dxdae0fdp            ZFe dxdae0fdq            ZGe dxdae0fdr            ZHe dxdae0fds            ZIe dxdae0fdt            ZJe dxdae0fdu            ZKdS )ya   LeViT

Paper: `LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference`
    - https://arxiv.org/abs/2104.01136

@article{graham2021levit,
  title={LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference},
  author={Benjamin Graham and Alaaeldin El-Nouby and Hugo Touvron and Pierre Stock and Armand Joulin and Herv'e J'egou and Matthijs Douze},
  journal={arXiv preprint arXiv:22104.01136},
  year={2021}
}

Adapted from official impl at https://github.com/facebookresearch/LeViT, original copyright bellow.

This version combines both conv/linear models and fixes torchscript compatibility.

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )OrderedDict)partial)DictListOptionalTupleUnionN)IMAGENET_DEFAULT_STDIMAGENET_DEFAULT_MEAN)	to_ntuple	to_2tupleget_act_layerDropPathtrunc_normal_ndgrid   )build_model_with_cfg)feature_take_indices)checkpoint_seq)generate_default_cfgsregister_modelLevitc                   X     e Zd Z	 d fd	Z ej                    d             Zd Z xZS )ConvNormr   r   c	           
         t                                                       t          j        |||||||d          | _        t          j        |          | _        t          j                            | j        j	        |           d S NFbias)
super__init__nnConv2dlinearBatchNorm2dbninit	constant_weight)
selfin_chsout_chskernel_sizestridepaddingdilationgroupsbn_weight_init	__class__s
            M/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/levit.pyr    zConvNorm.__init__,   sp    ifgxY_fklll.))
$'..99999    c           	      >   | j         | j        }}|j        |j        |j        z   dz  z  }|j        |d d d d d f         z  }|j        |j        |j        z  |j        |j        z   dz  z  z
  }t          j        |	                    d          |	                    d          |j
        dd          | j         j        | j         j        | j         j        | j         j                  }|j        j                            |           |j        j                            |           |S )N      ?r   r      )r-   r.   r/   r0   )r#   r%   r(   running_varepsr   running_meanr!   r"   sizeshaper-   r.   r/   r0   datacopy_)r)   cr%   wbms         r3   fusezConvNorm.fuse4   s   TW2I"&0S88HqD$,--Gbo	1R^bf5LQT4TTTIFF1IIqvvayy!'!""+dk6HK'$+2Ft{Oac c c 	
A	!r4   c                 R    |                      |                     |                    S N)r%   r#   r)   xs     r3   forwardzConvNorm.forwardA   s    wwt{{1~~&&&r4   )r   r   r   r   r   r   	__name__
__module____qualname__r    torchno_gradrC   rH   __classcell__r2   s   @r3   r   r   +   sr        lm: : : : : : U]__
 
 _
' ' ' ' ' ' 'r4   r   c                   V     e Zd Zd fd	Z ej                    d             Zd Z xZS )
LinearNormr   c                    t                                                       t          j        ||d          | _        t          j        |          | _        t          j                            | j        j	        |           d S r   )
r   r    r!   Linearr#   BatchNorm1dr%   r&   r'   r(   )r)   in_featuresout_featuresr1   r2   s       r3   r    zLinearNorm.__init__F   sd    i\FFF...
$'..99999r4   c                    | j         | j        }}|j        |j        |j        z   dz  z  }|j        |d d d f         z  }|j        |j        |j        z  |j        |j        z   dz  z  z
  }t          j        |	                    d          |	                    d                    }|j        j
                            |           |j        j
                            |           |S )Nr6   r   r   )r#   r%   r(   r8   r9   r   r:   r!   rT   r;   r=   r>   )r)   lr%   r@   rA   rB   s         r3   rC   zLinearNorm.fuseM   s    TW2I"&0S88HqDz!Gbo	1R^bf5LQT4TTTIaffQii++	A	!r4   c                     |                      |          }|                     |                    dd                                        |          S )Nr   r   )r#   r%   flatten
reshape_asrF   s     r3   rH   zLinearNorm.forwardX   s=    KKNNwwqyyA''221555r4   r   rI   rP   s   @r3   rR   rR   E   so        : : : : : : U]__  _6 6 6 6 6 6 6r4   rR   c                   V     e Zd Zd fd	Z ej                    d             Zd Z xZS )
NormLinearT{Gz?        c                    t                                                       t          j        |          | _        t          j        |          | _        t          j        |||          | _        t          | j        j
        |           | j        j        ,t          j                            | j        j        d           d S d S )Nr   )stdr   )r   r    r!   rU   r%   DropoutdroprT   r#   r   r(   r   r&   r'   )r)   rV   rW   r   rc   re   r2   s         r3   r    zNormLinear.__init__^   s    .--Jt$$	i\EEEdk(c2222;'Gdk.22222 ('r4   c                 x   | j         | j        }}|j        |j        |j        z   dz  z  }|j        | j         j        | j         j        z  |j        |j        z   dz  z  z
  }|j        |d d d f         z  }|j        || j        j        j        z  }n4|j        |d d d f         z                      d          | j        j        z   }t          j
        |                    d          |                    d                    }|j        j                            |           |j        j                            |           |S )Nr6   r   r   )r%   r#   r(   r8   r9   r   r:   Tviewr!   rT   r;   r=   r>   )r)   r%   rY   r@   rA   rB   s         r3   rC   zNormLinear.fuseh   s   AI"&0S88Gdg*TW^;r~PRPV?V[^>^^^Hqqqqz!6>DK&((AAAaaagJ&,,R004;3CCAIaffQii++	A	!r4   c                 x    |                      |                     |                     |                              S rE   )r#   re   r%   rF   s     r3   rH   zNormLinear.forwardw   s*    {{499TWWQZZ00111r4   )Tr`   ra   rI   rP   s   @r3   r_   r_   ]   so        3 3 3 3 3 3 U]__  _2 2 2 2 2 2 2r4   r_   c                        e Zd Z fdZ xZS )Stem8c           
         t                                                       d| _        |                     dt	          ||dz  ddd                     |                     d |                       |                     d	t	          |dz  |dz  ddd                     |                     d
 |                       |                     dt	          |dz  |ddd                     d S )N   conv1      r7   r   r-   r.   act1conv2act2conv3r   r    r-   
add_moduler   r)   r*   r+   	act_layerr2   s       r3   r    zStem8.__init__|   s    &'Q,!UV!W!W!WXXX		,,,'Q,1aPQ[\!]!]!]^^^		,,,'Q,1VW!X!X!XYYYYYr4   rJ   rK   rL   r    rO   rP   s   @r3   rl   rl   {   sA        Z Z Z Z Z Z Z Z Zr4   rl   c                        e Zd Z fdZ xZS )Stem16c           
      n   t                                                       d| _        |                     dt	          ||dz  ddd                     |                     d |                       |                     d	t	          |dz  |d
z  ddd                     |                     d |                       |                     dt	          |d
z  |dz  ddd                     |                     d |                       |                     dt	          |dz  |ddd                     d S )N   ro   rn   rq   r7   r   rr   rs   rt   rp   ru   rv   act3conv4rw   ry   s       r3   r    zStem16.__init__   s.   &'Q,!UV!W!W!WXXX		,,,'Q,1aPQ[\!]!]!]^^^		,,,'Q,1aPQ[\!]!]!]^^^		,,,'Q,1VW!X!X!XYYYYYr4   r{   rP   s   @r3   r}   r}      sA        
Z 
Z 
Z 
Z 
Z 
Z 
Z 
Z 
Zr4   r}   c                   &     e Zd Zd fd	Zd Z xZS )
DownsampleFc                     t                                                       || _        t          |          | _        |rt          j        d|dd          nd | _        d S )Nrq   r   F)r-   r.   count_include_pad)r   r    r-   r   
resolutionr!   	AvgPool2dpool)r)   r-   r   use_poolr2   s       r3   r    zDownsample.__init__   sY    #J//ZblBL61PUVVVVhl			r4   c                 n   |j         \  }}}|                    || j        d         | j        d         |          }| j        B|                     |                    dddd                                        dddd          }n|d d d d | j        d d | j        f         }|                    |d|          S )Nr   r   rq   r7   rg   )r<   ri   r   r   permuter-   reshape)r)   rG   BNCs        r3   rH   zDownsample.forward   s    '1aFF1doa($/!*<a@@9 		!))Aq!Q//0088Aq!DDAA!!!]]t{]MMdkM12AyyB"""r4   F)rJ   rK   rL   r    rH   rO   rP   s   @r3   r   r      sR        m m m m m m# # # # # # #r4   r   c                        e Zd ZU eeej        f         ed<   ddddej	        f fd	Z
 ej                    d fd	            Zd	ej        d
ej        fdZd Z xZS )	Attentionattention_bias_cachern         @   Fc                    t                                                       |rt          nt          }t	          |          }|| _        || _        |dz  | _        || _        ||z  | _	        t          ||z            | _        t          ||z            |z  | _         ||| j        | j	        dz  z             | _        t          j        t!          d |            fd || j        |d          fg                    | _        t          j        t'          j        ||d         |d         z                      | _        t'          j        t/          t'          j        |d                   t'          j        |d                                                           d          }	|	dd d d f         |	dd d d f         z
                                  }
|
d         |d         z  |
d         z   }
|                     d	|
d
           i | _        d S )N      r7   actlnr   r1   r   .attention_bias_idxsF
persistent)r   r    r   rR   r   use_conv	num_headsscalekey_dimkey_attn_dimintval_dimval_attn_dimqkvr!   
Sequentialr   proj	ParameterrM   zerosattention_biasesstackr   aranger[   absregister_bufferr   )r)   dimr   r   
attn_ratior   r   rz   ln_layerposrel_posr2   s              r3   r    zAttention.__init__   s    	'788Zz**
 "_
#i/:/00
W 455	A8C!2T5F5J!JKKM+IIKK 88D-s1EEEF/
 # #  	
 !#U[JqMT^_`TaDa-b-b c ck&jm!<!<el:VW=>Y>YZZ[[ccdeffsAAAt|$s3aaa<'88==??1:
1-;2GNNN$&!!!r4   Tc                 r    t                                          |           |r| j        ri | _        d S d S d S rE   r   trainr   r)   moder2   s     r3   r   zAttention.train   M    d 	+D- 	+(*D%%%	+ 	+ 	+ 	+r4   devicereturnc                     t           j                                        s| j        r| j        d d | j        f         S t          |          }|| j        vr| j        d d | j        f         | j        |<   | j        |         S rE   rM   jit
is_tracingtrainingr   r   strr   r)   r   
device_keys      r3   get_attention_biaseszAttention.get_attention_biases       9!! 	9T] 	9(D,D)DEEVJ!:::8<8MaaaQUQiNi8j)*5,Z88r4   c                 V   | j         r|j        \  }}}}|                     |                              || j        d||z                                | j        | j        | j        gd          \  }}}|                    dd          |z  | j	        z  | 
                    |j                  z   }	|	                    d          }	||	                    dd          z                      |d||          }n|j        \  }}
}|                     |                              ||
| j        d                              | j        | j        | j        gd          \  }}}|                    dddd          }|                    dddd          }|                    dddd          }||z  | j	        z  | 
                    |j                  z   }	|	                    d          }	|	|z                      dd                              ||
| j                  }|                     |          }|S )Nrg   r7   r   rq   r   r   )r   r<   r   ri   r   splitr   r   	transposer   r   r   softmaxr   r   r   r   )r)   rG   r   r   HWqkvattnr   s              r3   rH   zAttention.forward   s   = 	LJAq!Qhhqkk&&4>2q1u. ..3eT\4<QUQ]4^dee.f.f Aq! KKB''!+tz9D<U<UVWV^<_<__D<<B<''DT^^B+++11!RA>>AAgGAq!hhqkk&&1dnb* **/%t|T\0Z`a%*b*b Aq!		!Q1%%A		!Q1%%A		!Q1%%Aq54:%(A(A!((K(KKD<<B<''D$$Q**221a9JKKAIIaLLr4   TrJ   rK   rL   r   r   rM   Tensor__annotations__r!   SiLUr    rN   r   r   r   rH   rO   rP   s   @r3   r   r      s         sEL01111 g!' !' !' !' !' !'F U]__+ + + + + _+
95< 9EL 9 9 9 9      r4   r   c                        e Zd ZU eeej        f         ed<   ddddddej	        f fd	Z
 ej                    d fd		            Zd
ej        dej        fdZd Z xZS )AttentionDownsampler   rn          @r7   r   Fc           
         t                                                       t          |          }|| _        || _        || _        || _        ||z  | _        t          ||z            | _	        | j	        | j        z  | _
        |dz  | _        || _        | j        r-t          }t          t          j        |	rdnd|	rdndd          }nt"          }t          t$          ||	          } ||| j
        | j        z             | _        t          j        t+          d ||	          fd
 ||| j                  fg                    | _        t          j        t+          d |
            fd
 || j
        |          fg                    | _        t          j        t3          j        ||d         |d         z                      | _        t3          j        t;          t3          j        |d                   t3          j        |d                                                           d          }t3          j        t;          t3          j        d|d         |          t3          j        d|d         |                                                  d          }|dd d d f         |dd d d f         z
                                   }|d         |d         z  |d         z   }| !                    d|d           i | _"        d S )Nr   rq   r   r   F)r,   r.   r   )r   r   down)r-   r   r   )step.r   r   )#r   r    r   r-   r   r   r   r   r   r   r   r   r   r   r   r!   r   rR   r   kvr   r   r   r   r   rM   r   r   r   r   r   r[   r   r   r   )r)   in_dimout_dimr   r   r   r-   r   r   r   rz   r   	sub_layerk_posq_posr   r2   s                   r3   r    zAttentionDownsample.__init__   s    	z**
$"#i/:/00 L4>9_
 = 	VH!)0AAqx:N!!Qbgi i iII "H
zHUUUI(64#4t7H#HII{YYf---.88FD$5667,
       M+IIKK 88D-w778/
 # #  	
 !#U[JqMT^_`TaDa-b-b c cF5<
1#>#>ZXY]@[@[\\]]eefghhFLJqM777LJqM777
 
   71:: 	 aaa&sD!!!|)<<AACC1:
1-;2GNNN$&!!!r4   Tc                 r    t                                          |           |r| j        ri | _        d S d S d S rE   r   r   s     r3   r   zAttentionDownsample.train1  r   r4   r   r   c                     t           j                                        s| j        r| j        d d | j        f         S t          |          }|| j        vr| j        d d | j        f         | j        |<   | j        |         S rE   r   r   s      r3   r   z(AttentionDownsample.get_attention_biases7  r   r4   c                 V   | j         r@|j        \  }}}}|dz
  | j        z  dz   |dz
  | j        z  dz   }}|                     |                              || j        d||z                                | j        | j        gd          \  }}	| 	                    |                              || j        | j        d          }
|

                    dd          |z  | j        z  |                     |j                  z   }|                    d          }|	|
                    dd          z                      || j        ||          }nK|j        \  }}}|                     |                              ||| j        d                              | j        | j        gd          \  }}	|                    dddd          }|	                    dddd          }	| 	                    |                              |d| j        | j                                      dddd          }
|
|z  | j        z  |                     |j                  z   }|                    d          }||	z  
                    dd                              |d| j                  }|                     |          }|S )Nr   rg   r7   r   r   rq   r   )r   r<   r-   r   ri   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r)   rG   r   r   r   r   HHWWr   r   r   r   r   s                r3   rH   zAttentionDownsample.forward@  sd   = 	MJAq!Q!e+a/!a%DK1G!1KB771::??1dnb!a%@@FFVZVbGcijFkkDAqq		q$.$,CCAKKB''!+tz9D<U<UVWV^<_<__D<<B<''DT^^B+++44Q8I2rRRAAgGAq!771::??1a<<BBDLRVR^C_efBggDAq		!Q1%%A		!Q1%%Aq		q"dndlCCKKAqRSUVWWAq54:%(A(A!((K(KKD<<B<''D$$Q**221b$:KLLAIIaLLr4   r   r   rP   s   @r3   r   r      s         sEL01111 g7' 7' 7' 7' 7' 7'r U]__+ + + + + _+
95< 9EL 9 9 9 9      r4   r   c                   >     e Zd ZdZdddej        df fd	Zd Z xZS )LevitMlpzL MLP for Levit w/ normalization + ability to switch btw conv and linear
    NFra   c                    t                                                       |p|}|p|}|rt          nt          } |||          | _         |            | _        t          j        |          | _         |||d          | _	        d S )Nr   r   )
r   r    r   rR   ln1r   r!   rd   re   ln2)	r)   rV   hidden_featuresrW   r   rz   re   r   r2   s	           r3   r    zLevitMlp.__init__]  s     	#2{)8['788Z8K999;;Jt$$	8O\!LLLr4   c                     |                      |          }|                     |          }|                     |          }|                     |          }|S rE   )r   r   re   r   rF   s     r3   rH   zLevitMlp.forwardp  sC    HHQKKHHQKKIIaLLHHQKKr4   )	rJ   rK   rL   __doc__r!   r   r    rH   rO   rP   s   @r3   r   r   Z  so         
 !gM M M M M M&      r4   r   c            	       B     e Zd Zdddej        dddddf	 fd	Zd	 Z xZS )
LevitDownsamplern   r   r   Nr   Fra   c                 4   t                                                       |p|}t          |||||||	|
|	  	        | _        t	          |t          ||z            |
|          | _        |dk    rt          |          nt          j	                    | _
        d S )N)	r   r   r   r   r   rz   r   r   r   r   rz   ra   )r   r    r   attn_downsampler   r   mlpr   r!   Identity	drop_path)r)   r   r   r   r   r   	mlp_ratiorz   attn_act_layerr   r   r   r   r2   s                r3   r    zLevitDownsample.__init__y  s     	'492!$!
 
 
 
 
 
 )#$$	
 
 
 1:B),,,BKMMr4   c                     |                      |          }||                     |                     |                    z   }|S rE   )r   r   r   rF   s     r3   rH   zLevitDownsample.forward  s:      ##txx{{+++r4   rJ   rK   rL   r!   r   r    rH   rO   rP   s   @r3   r   r   x  sr         g$R $R $R $R $R $RL      r4   r   c                   @     e Zd Zdddddej        ddf fd	Zd	 Z xZS )

LevitBlockrn   r   r   r   FNra   c           	         t                                                       |	p|}	t          |||||||	          | _        |
dk    rt	          |
          nt          j                    | _        t          |t          ||z            ||          | _
        |
dk    rt	          |
          nt          j                    | _        d S )N)r   r   r   r   r   r   rz   ra   r   )r   r    r   r   r   r!   r   
drop_path1r   r   r   
drop_path2)r)   r   r   r   r   r   r   r   rz   r   r   r2   s              r3   r    zLevitBlock.__init__  s     	'49!!$  	 2;R(9---R[]]i  	
 
 
 2;R(9---R[]]r4   c                     ||                      |                     |                    z   }||                     |                     |                    z   }|S rE   )r   r   r  r   rF   s     r3   rH   zLevitBlock.forward  sG    		!---,,,r4   r   rP   s   @r3   r   r     so        
 g!S !S !S !S !S !SF      r4   r   c            
       D     e Zd Zddddej        dddddf
 fd		Zd
 Z xZS )
LevitStagerp   rn   r   Nr    Fra   c                    t                                                       t          |
          }
|r/t          |||||z  dd||	|
||          | _        d |
D             }
n ||k    sJ t          j                    | _        g }t          |          D ]}|t          |||||||	|
||
  
        gz  } t          j	        | | _
        d S )Nr   r   )	r   r   r   r   rz   r   r   r   r   c                 $    g | ]}|d z
  dz  d z   S )r   r7    ).0rs     r3   
<listcomp>z'LevitStage.__init__.<locals>.<listcomp>  s%    ???q1q5Q,*???r4   )r   r   r   rz   r   r   r   r   )r   r    r   r   
downsampler!   r   ranger   r   blocks)r)   r   r   r   depthr   r   r   rz   r   r   r  r   r   r  _r2   s                   r3   r    zLevitStage.__init__  s     	z**
 	,- G+#-%!#  DO @?J???JJW$$$$ kmmDOu 	 	Az#%##-%!#    FF mV,r4   c                 Z    |                      |          }|                     |          }|S rE   )r  r  rF   s     r3   rH   zLevitStage.forward  s'    OOAKKNNr4   r   rP   s   @r3   r  r    so         g4- 4- 4- 4- 4- 4-l      r4   r  c                   .    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d+ fd	Zej        j        d             Zej        j        d,d            Z	ej        j        d-d            Z
ej        j        dej        fd            Zd.dedee         fdZ	 	 	 	 	 d/dej        deeeee         f                  deded ed!edeeej                 eej        eej                 f         f         fd"Z	 	 	 d0deeee         f         d$ed%efd&Zd' Zd,d(efd)Zd* Z xZS )1r   z Vision Transformer with support for patch or hybrid CNN input stage

    NOTE: distillation is defaulted to True since pretrained weights use it, will cause problems
    w/ train scripts that don't take tuple outputs,
       rq        @      rq   r   Ns16	subsample
hard_swishFavgra   c                 J   t                                                       t          |          }t          |p|          }|| _        || _        || _        |d         x| _        | _        || _        || _	        d| _
        g | _        t          |          }t          |          |k    sJ  t          |          |          } t          |          |          } t          |          |	          }	|
|dk    sJ |
| _        |}nS|dv sJ |dk    rt          ||d         |          | _        nt!          ||d         |          | _        | j        j        }t%          d t'          t)          |          t)          |                    D                       }|d         }g }t+          |          D ]}|dk    rdnd	|t-          |||         |||         ||         ||         |	|         ||||dk    r|nd
|          gz  }|z  }t%          fd|D                       }| xj        t/          ||         |d|           gz  c_        ||         }t1          j        | | _        |dk    rt7          |d         ||          nt1          j                    | _        d S )Nrg   Fr7   )r  s8r  r   )rz   c                     g | ]
\  }}||z  S r  r  )r	  ips      r3   r  z"Levit.__init__.<locals>.<listcomp>E  s     [[[tq!AF[[[r4   r   r  )
r  r   r   r   rz   r   r   r   r  r   c                 &    g | ]}|d z
  z  d z   S r]   r  )r	  r
  stage_strides     r3   r  z"Levit.__init__.<locals>.<listcomp>[  s'    PPPQ< 7! ;PPPr4   zstages.)num_chs	reductionmodulere   )r   r    r   r   num_classesglobal_poolnum_featureshead_hidden_size	embed_dim	drop_rategrad_checkpointingfeature_infolenr   stemr}   rl   r-   tuplezipr   r  r  dictr!   r   stagesr_   r   head)r)   img_sizein_chansr*  r.  r   r  r   r   r   stem_backbonestem_stride	stem_typedown_oprz   r   r   r+  r/  drop_path_rate
num_stagesr-   r   r   r7  r"  r%  r2   s                             @r3   r    zLevit.__init__  s   * 	!),,	&~'BCC &&4=bMAD1"""'^^
5zzZ'''')Ij)))44	*Yz**:66
)Ij)))44	$!####%DI FF----E!!"8Yq\YOOO		!(IaLINNN	Y%F[[s9X3F3F	RXHYHY/Z/Z[[[\\
1z"" 	" 	"A !A111Lz!Ah#A,%a=#A,#-%!&2a&7&777R(    F l"FPPPPZPPPQQJ$y|vVc`aVcVc"d"d"d!eeq\FFmV, OZ\]ooJy}k	JJJJcecncpcp			r4   c                 b    d |                                                                  D             S )Nc                     h | ]}d |v |	S )r   r  )r	  rG   s     r3   	<setcomp>z(Levit.no_weight_decay.<locals>.<setcomp>e  s#    OOOa7IQ7N7N7N7N7Nr4   )
state_dictkeysr)   s    r3   no_weight_decayzLevit.no_weight_decayc  s,    OO4??,,1133OOOOr4   c                 ,    t          dddg          }|S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )r3  r  )r6  )r)   coarsematchers      r3   group_matcherzLevit.group_matcherg  s)    4-/CD
 
 
 r4   Tc                     || _         d S rE   )r0  r)   enables     r3   set_grad_checkpointingzLevit.set_grad_checkpointingo      "(r4   r   c                     | j         S rE   )r8  rF  s    r3   get_classifierzLevit.get_classifiers  s
    yr4   r*  r+  c                     || _         ||| _        |dk    rt          | j        || j                  nt          j                    | _        d S Nr   r)  )r*  r+  r_   r,  r/  r!   r   r8  r)   r*  r+  s      r3   reset_classifierzLevit.reset_classifierw  sb    &"*DDORSOO {A A A AY[YdYfYf 				r4   NCHWrG   indicesnorm
stop_early
output_fmtintermediates_onlyc           	         |dv s
J d            g }t          t          | j                  |          \  }}	|                     |          }|j        \  }
}}}| j        s)|                    d                              dd          }t          j	        
                                s|s| j        }n| j        d|	dz            }t          |          D ]\  }} ||          }||v r^| j        r|                    |           nA|                    |                    |
||d                              dddd                     |dz   dz
  dz  }|dz   dz
  dz  }|r|S ||fS )	a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )rW  zOutput shape must be NCHW.r7   r   Nrg   r   rq   )r   r2  r7  r3  r<   r   r[   r   rM   r   is_scripting	enumerateappendr   r   )r)   rG   rX  rY  rZ  r[  r\  intermediatestake_indices	max_indexr   r   r   r   r7  feat_idxstages                    r3   forward_intermediateszLevit.forward_intermediates~  s   * Y&&&(D&&&"6s4;7G7G"Q"Qi IIaLLW
1a} 	-		!&&q!,,A9!!## 	1: 	1[FF[)a-0F(00 	! 	!OHeaA<''= U!((++++!((1aB)?)?)G)G1aQR)S)STTTQq AQq AA 	!  -r4   r   
prune_norm
prune_headc                     t          t          | j                  |          \  }}| j        d|dz            | _        |r|                     dd           |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r  )r   r2  r7  rV  )r)   rX  rg  rh  rb  rc  s         r3   prune_intermediate_layerszLevit.prune_intermediate_layers  s\     #7s4;7G7G"Q"Qik.9q=.1 	)!!!R(((r4   c                 0   |                      |          }| j        s)|                    d                              dd          }| j        r4t
          j                                        st          | j	        |          }n| 	                    |          }|S )Nr7   r   )
r3  r   r[   r   r0  rM   r   r^  r   r7  rF   s     r3   forward_featureszLevit.forward_features  s    IIaLL} 	-		!&&q!,,A" 	59+A+A+C+C 	t{A..AAAAr4   
pre_logitsc                     | j         dk    r3| j        r|                    d          n|                    d          }|r|n|                     |          S )Nr  r   rg   r   r   )r+  r   meanr8  )r)   rG   rm  s      r3   forward_headzLevit.forward_head  sT    u$$(,H8$$$166a6==A0qqDIIaLL0r4   c                 Z    |                      |          }|                     |          }|S rE   )rl  rq  rF   s     r3   rH   zLevit.forward  s-    !!!$$a  r4   )r  rq   r  r  r  r  r  r   r   NNr  r  r  NFr  ra   ra   r   r   rE   )NFFrW  F)r   FT)rJ   rK   rL   r   r    rM   r   ignorerG  rK  rO  r!   ModulerR  r   r   r   rV  r   r	   r   boolr   rf  rj  rl  rq  rH   rO   rP   s   @r3   r   r     s         ")Nq Nq Nq Nq Nq Nq` YP P P Y    Y) ) ) ) Y	    g gC gx} g g g g 8<$$',0  0 |0  eCcN340  	0 
 0  0  !%0  
tEL!5tEL7I)I#JJ	K0  0  0  0 h ./$#	 3S	>*  	     1 1$ 1 1 1 1
      r4   c                        e Zd Z fdZej        j        dej        fd            Z	dde
dee         fdZej        j        dd	            ZddefdZ xZS )LevitDistilledc                      t                      j        |i | | j        dk    rt          | j        | j                  nt          j                    | _        d| _        d S )Nr   F)	r   r    r*  r_   r,  r!   r   	head_distdistilled_training)r)   argskwargsr2   s      r3   r    zLevitDistilled.__init__  sa    $)&)))LPL\_`L`L`D$5t7GHHHfhfqfsfs"'r4   r   c                     | j         | j        fS rE   )r8  ry  rF  s    r3   rR  zLevitDistilled.get_classifier  s    y$.((r4   Nr*  r+  c                     || _         ||| _        |dk    rt          | j        || j                  nt          j                    | _        |dk    rt          | j        |          nt          j                    | _        d S rT  )	r*  r+  r_   r,  r/  r!   r   r8  ry  rU  s      r3   rV  zLevitDistilled.reset_classifier  s    &"*DDORSOO {A A A AY[YdYfYf 		GRUVD$5{CCC\^\g\i\ir4   Tc                     || _         d S rE   )rz  rM  s     r3   set_distilled_trainingz%LevitDistilled.set_distilled_training  rP  r4   Frm  c                 J   | j         dk    r3| j        r|                    d          n|                    d          }|r|S |                     |          |                     |          }}| j        r)| j        r"t          j        	                                s||fS ||z   dz  S )Nr  ro  r   r   r7   )
r+  r   rp  r8  ry  rz  r   rM   r   r^  )r)   rG   rm  x_dists       r3   rq  zLevitDistilled.forward_head  s    u$$(,H8$$$166a6==A 	HIIaLL$.."3"36" 	$t} 	$UY=S=S=U=U 	$f9 J!##r4   rE   r   r   )rJ   rK   rL   r    rM   r   rs  r!   rt  rR  r   r   r   rV  r  ru  rq  rO   rP   s   @r3   rw  rw    s        ( ( ( ( (
 Y)	 ) ) ) )j jC jhsm j j j j Y) ) ) )$ $$ $ $ $ $ $ $ $ $r4   rw  c                    d| v r| d         } d |                                  D             } |                                }i }t          |                                |                                 |                                |                                           D ]L\  }}}}|j        dk    r|j        dk    r|d d d d d d f         }|j        |j        k    r
d|v sd|v sJ |||<   M|S )Nmodelc                 "    i | ]\  }}d |v	||S )r   r  )r	  r   r   s      r3   
<dictcomp>z(checkpoint_filter_fn.<locals>.<dictcomp>  s*    XXX41a9NVW9W9W!Q9W9W9Wr4   rp   r7   r8  stem.conv1.linear)itemsrD  r5  rE  valuesndimr<   )rD  r  Dout_dictkakbvavbs           r3   checkpoint_filter_fnr    s   *(
 YX:#3#3#5#5XXXJAHaffhh
(9(9188::zGXGXGZGZ[[  BB7a<<BGqLLAAAqqq$$%B8rxR<<#6"#<#<#<#<Or4   )        r   )rp      rn   )r7   rq   rp   )r.  r   r   r  )rp   rn   r  )rp   rp   rp   )r  i   r      )rq      r  )r  r     )r  r     )r  	   r  silur   )r.  r   r   r  rz   r=  )r    i  r  )rn   
   r   )r  r  i   )rn   r  r   )r.  r   r   r  rz   )rp   rn   r  )r  r  r  )rn   r  r  )

levit_128s	levit_128	levit_192	levit_256	levit_384levit_384_s8levit_512_s8	levit_512
levit_256d
levit_512dFTc                 z   d| v }|                     dd          }|                    dd          r|s|                    dd           |$| t          v r| }n|r|                     dd          }t          t          |         fi |}t          |rt          nt          | |ft          t          d	|
          d|}|S )N_convout_indices)r   r   r7   features_onlyFfeature_clsgetterr  T)flatten_sequentialr  )pretrained_filter_fnfeature_cfg)
popget
setdefault
model_cfgsreplacer6  r   rw  r   r  )	variantcfg_variant
pretrained	distilledr|  is_convr  	model_cfgr  s	            r3   create_levitr  +  s     G**]I66Kzz/5)) 3' 3-222j  !KK 	7!//'266KZ,7777I #. 2DkJJJ   E Lr4   r  c                 6    | ddd dddt           t          ddd|S )	Nr  )rq   r  r  g?bicubicTr  )head.linearzhead_dist.linear)urlr*  
input_size	pool_sizecrop_pctinterpolationfixed_input_sizerp  rc   
first_conv
classifier)r   r
   )r  r|  s     r3   _cfgr  B  s8    =t%.B)9\   r4   zlevit_128s.fb_dist_in1kztimm/)	hf_hub_idzlevit_128.fb_dist_in1kzlevit_192.fb_dist_in1kzlevit_256.fb_dist_in1kzlevit_384.fb_dist_in1kzlevit_conv_128s.fb_dist_in1k)rp   rp   )r  r  zlevit_conv_128.fb_dist_in1kzlevit_conv_192.fb_dist_in1kzlevit_conv_256.fb_dist_in1kzlevit_conv_384.fb_dist_in1kzlevit_384_s8.untrainedr  )r  zlevit_512_s8.untrainedzlevit_512.untrainedzlevit_256d.untrainedzlevit_512d.untrainedzlevit_conv_384_s8.untrainedzlevit_conv_512_s8.untrained)zlevit_conv_512.untrainedzlevit_conv_256d.untrainedzlevit_conv_512d.untrainedr   c                     t          dd| i|S )Nr  r  )r  r  r  r|  s     r3   r  r    s    FFFvFFFr4   c                     t          dd| i|S )Nr  r  )r  r  r  s     r3   r  r        EE
EfEEEr4   c                     t          dd| i|S )Nr  r  )r  r  r  s     r3   r  r    r  r4   c                     t          dd| i|S )Nr  r  )r  r  r  s     r3   r  r    r  r4   c                     t          dd| i|S )Nr  r  )r  r  r  s     r3   r  r    r  r4   c                     t          dd| i|S )Nr  r  )r  r  r  s     r3   r  r    s    HH:HHHHr4   c                      t          d| dd|S )Nr  Fr  r  )r  r  r  s     r3   r  r    s    Y:YYRXYYYr4   c                      t          d| dd|S )Nr  Fr  )r  r  r  s     r3   r  r    s    V
eVVvVVVr4   c                      t          d| dd|S )Nr  Fr  )r  r  r  s     r3   r  r        WuWWPVWWWr4   c                      t          d| dd|S )Nr  Fr  )r  r  r  s     r3   r  r    r  r4   c                      t          d| dd|S )Nlevit_conv_128sTr  r   )r  r  r  s     r3   r  r    s    Zj4ZZSYZZZr4   c                      t          d| dd|S )Nlevit_conv_128Tr  )r  r  r  s     r3   r  r        YZ$YYRXYYYr4   c                      t          d| dd|S )Nlevit_conv_192Tr  )r  r  r  s     r3   r  r    r  r4   c                      t          d| dd|S )Nlevit_conv_256Tr  )r  r  r  s     r3   r  r    r  r4   c                      t          d| dd|S )Nlevit_conv_384Tr  )r  r  r  s     r3   r  r    r  r4   c                      t          d| dd|S )Nlevit_conv_384_s8Tr  )r  r  r  s     r3   r  r    s    \
T\\U[\\\r4   c                 "    t          d| ddd|S )Nlevit_conv_512_s8TFr  r   r  )r  r  r  s     r3   r  r    s!    m
T]bmmflmmmr4   c                 "    t          d| ddd|S )Nlevit_conv_512TFr  )r  r  r  s     r3   r  r    s!    jZ$Z_jjcijjjr4   c                 "    t          d| ddd|S )Nlevit_conv_256dTFr  )r  r  r  s     r3   r  r    !    kj4[`kkdjkkkr4   c                 "    t          d| ddd|S )Nlevit_conv_512dTFr  )r  r  r  s     r3   r  r    r  r4   )NFT)r  r   )Lr   collectionsr   	functoolsr   typingr   r   r   r   r	   rM   torch.nnr!   	timm.datar
   r   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__rt  r   rR   r_   r   rl   r}   r   r   r   r   r   r   r  r   rw  r  r6  r  r  r  default_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r4   r3   <module>r     s   2 $ # # # # #       5 5 5 5 5 5 5 5 5 5 5 5 5 5        A A A A A A A A \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ * * * * * * + + + + + + ' ' ' ' ' ' < < < < < < < <)' ' ' ' 'ry ' ' '46 6 6 6 6 6 6 602 2 2 2 2 2 2 2<	Z 	Z 	Z 	Z 	ZBM 	Z 	Z 	ZZ Z Z Z ZR] Z Z Z# # # # # # # #"L L L L L	 L L L^b b b b b") b b bJ    ry   <* * * * *bi * * *Z' ' ' ' ' ' ' 'T: : : : : : : :zD D D D DBI D D DN!$ !$ !$ !$ !$U !$ !$ !$H  ( Tt!2)U U Ud!29V V Vd!2)U U Ud!2)U U Ud!29V V V !29D* * * !2ID* * *
 d"B+Ybhj j j t!2)_eg g gt!2Iagi i i7  
@   .    %$ 3&tt     3&
 dd  3& dd  3& dd  3& dd  3&& #DD% % %'3&. "44$ $ $/3&6 "44$ $ $73&> "44$ $ $?3&F "44$ $ $G3&P ddm<<<Q3&R ddm<<<S3&T 44=999U3&V DDM:::W3&X DDM:::Y3&\ "44=#A#A#A]3&^ "44=#A#A#A_3&` !% > > >!%!?!?!?!%!?!?!?e3& 3& 3& 3 3l G Ge G G G G F FU F F F F F FU F F F F F FU F F F F F FU F F F F I I I I I I Z Z Z Z Z Z W WU W W W W X Xe X X X X X Xe X X X X [ [5 [ [ [ [ Z Z% Z Z Z Z Z Z% Z Z Z Z Z Z% Z Z Z Z Z Z% Z Z Z Z ] ]U ] ] ] ] n nU n n n n k k% k k k k l l5 l l l l l l5 l l l l l lr4   