
    Ngfu                     b   d Z ddlmZmZmZmZ ddlZddlmZ ddl	mc m
Z ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZmZ d	gZ G d
 dej                  Z G d dej                  Z G d dej                  Z  G d dej                  Z! G d dej                  Z" G d d	ej                  Z#d Z$d Z%d Z&d,dZ'd-dZ( e e(d           e(d           e(d           e(d           e(d           e(d           e(d           e(dddd !          d"          Z)ed.d#e#fd$            Z*ed.d#e#fd%            Z+ed.d#e#fd&            Z,ed.d#e#fd'            Z-ed.d#e#fd(            Z.ed.d#e#fd)            Z/ed.d#e#fd*            Z0ed.d#e#fd+            Z1dS )/z 
CoaT architecture.

Paper: Co-Scale Conv-Attentional Image Transformers - https://arxiv.org/abs/2104.06399

Official CoaT code at: https://github.com/mlpc-ucsd/CoaT

Modified from timm/models/vision_transformer.py
    )ListOptionalTupleUnionNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPath	to_2tupletrunc_normal__assert	LayerNorm   )build_model_with_cfg)register_modelgenerate_default_cfgsCoaTc                   >     e Zd ZdZ fdZdeeef         fdZ xZS )ConvRelPosEncz+ Convolutional relative position encoding. c           
      d   t                                                       t          |t                    r||i}|| _        n+t          |t
                    r|| _        nt                      t          j                    | _	        g | _
        |                                D ]u\  }}d}||dz
  |dz
  z  z   dz  }t          j        |z  |z  ||f||f||f|z            }| j	                            |           | j
                            |           vfd| j
        D             | _        dS )aj  
        Initialization.
            Ch: Channels per head.
            h: Number of heads.
            window: Window size(s) in convolutional relative positional encoding. It can have two forms:
                1. An integer of window size, which assigns all attention heads with the same window s
                    size in ConvRelPosEnc.
                2. A dict mapping window size to #attention head splits (
                    e.g. {window size 1: #attention head split 1, window size 2: #attention head split 2})
                    It will apply different window size to the attention head splits.
        r      )kernel_sizepaddingdilationgroupsc                     g | ]}|z  S  r   ).0xhead_chss     L/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/coat.py
<listcomp>z*ConvRelPosEnc.__init__.<locals>.<listcomp>B   s    FFFq8|FFF    N)super__init__
isinstanceintwindowdict
ValueErrornn
ModuleList	conv_listhead_splitsitemsConv2dappendchannel_splits)
selfr"   	num_headsr*   
cur_windowcur_head_splitr   padding_sizecur_conv	__class__s
    `       r#   r'   zConvRelPosEnc.__init__   sW    	fc"" 	i(F DKK%% 	 DKK,,*0,,.. 	4 	4&JH '*q.X\)JJqPLy))'4%|4"H-%0  H N!!(+++##N3333FFFFT5EFFFr%   sizec                    |j         \  }}}}|\  }}	t          |d||	z  z   k    d           |d d d d dd d d f         }
|d d d d dd d d f         }|                    dd                              |||z  ||	          }t	          j        || j        d          }g }t          | j                  D ])\  }}|	                     |||                              *t	          j
        |d          }|                    |||||	z                                dd          }|
|z  }t          j        |d          }|S )Nr    dim)r   r   r   r   r   r   )shaper   	transposereshapetorchsplitr4   	enumerater/   r3   catFpad)r5   qvr<   Br6   NCHWq_imgv_img
v_img_listconv_v_img_listiconv
conv_v_imgEV_hats                     r#   forwardzConvRelPosEnc.forwardD   sh   W9a1QQY### !!!QQQAAA+!!!QQQAAA+B''//9q=!QGG[(;CCC
 00 	8 	8GAt""44
1#6#67777YA666
''9aQ??II"bQQ
#v122r%   	__name__
__module____qualname____doc__r'   r   r)   r[   __classcell__r;   s   @r#   r   r      sg        55(G (G (G (G (GT%S/        r%   r   c                   J     e Zd ZdZ	 	 	 	 	 d	 fd	Zdeeef         fdZ xZS )
FactorAttnConvRelPosEnczK Factorized attention with convolutional relative position encoding class.    F        Nc                 X   t                                                       || _        ||z  }|dz  | _        t	          j        ||dz  |          | _        t	          j        |          | _        t	          j        ||          | _	        t	          j        |          | _
        || _        d S )Ng         )bias)r&   r'   r6   scaler-   LinearqkvDropout	attn_dropproj	proj_dropcrpe)	r5   rB   r6   qkv_biasrn   rp   shared_crpehead_dimr;   s	           r#   r'   z FactorAttnConvRelPosEnc.__init__\   s     	")#%
9S#'999I..Ic3''	I..  			r%   r<   c                 J   |j         \  }}}|                     |                              ||d| j        || j        z                                ddddd          }|                    d          \  }}}	|                    d          }
|
                    dd          |	z  }||z  }|                     ||	|	          }| j	        |z  |z   }|                    dd                              |||          }| 
                    |          }|                     |          }|S )
Nrh   r   r   r      rA   r?   r@   r<   )rC   rl   rE   r6   permuteunbindsoftmaxrD   rq   rj   ro   rp   )r5   r!   r<   rN   rO   rP   rl   rL   krM   	k_softmax
factor_attrq   s                r#   r[   zFactorAttnConvRelPosEnc.forwardr   s   '1a hhqkk!!!Q4>1;NOOWWXY[\^_abdeff**Q--1a II!I$$	((R0014
^
 yyADy)) J#d*KK1%%aA.. IIaLLNN1r%   )re   Frf   rf   Nr\   rb   s   @r#   rd   rd   Z   sw        UU            ,uS#X        r%   rd   c                   @     e Zd ZdZd fd	Zdeeef         fdZ xZS )
ConvPosEnczz Convolutional Position Encoding. 
        Note: This module is similar to the conditional position encoding in CPVT.
    rh   c                     t          t          |                                            t          j        |||d|dz  |          | _        d S )Nr   r   )r   )r&   r   r'   r-   r2   ro   )r5   rB   r{   r;   s      r#   r'   zConvPosEnc.__init__   sE    j$((***Ic31ad3???			r%   r<   c                    |j         \  }}}|\  }}t          |d||z  z   k    d           |d d d df         |d d dd f         }	}|	                    dd                              ||||          }
|                     |
          |
z   }|                    d                              dd          }t          j        ||fd          }|S )Nr   r>   r   rA   )rC   r   rD   viewro   flattenrF   rI   )r5   r!   r<   rN   rO   rP   rQ   rR   	cls_token
img_tokensfeats              r#   r[   zConvPosEnc.forward   s    '1a1QQY### !"!!!RaR%!AAAqrrE(:	 ##Aq))..q!Q::IIdOOd"IIaLL""1a(( Iy!n!,,,r%   )rh   r\   rb   s   @r#   r   r      sp         @ @ @ @ @ @uS#X        r%   r   c            	       f     e Zd ZdZdddddej        ej        ddf	 fd	Zdee	e	f         fdZ
 xZS )	SerialBlockz Serial block class.
        Note: In this implementation, each serial block only contains a conv-attention and a FFN (MLP) module.       @Frf   Nc                 x   t                                                       |
| _         |	|          | _        t	          ||||||          | _        |dk    rt          |          nt          j                    | _	         |	|          | _
        t          ||z            }t          ||||          | _        d S )Nr6   rr   rn   rp   rs   rf   in_featureshidden_features	act_layerdrop)r&   r'   cpenorm1rd   factoratt_crper   r-   Identity	drop_pathnorm2r)   r   mlp)r5   rB   r6   	mlp_ratiorr   rp   rn   r   r   
norm_layer
shared_cpers   mlp_hidden_dimr;   s                r#   r'   zSerialBlock.__init__   s     	 Z__
5#
 
 
 1:B),,,BKMM  Z__
S9_--*	
 
 
r%   r<   c                 <   |                      ||          }|                     |          }|                     ||          }||                     |          z   }|                     |          }|                     |          }||                     |          z   }|S N)r   r   r   r   r   r   )r5   r!   r<   curs       r#   r[   zSerialBlock.forward   s    HHQjjmm!!#t,,s### jjmmhhsmms###r%   )r]   r^   r_   r`   r-   GELUr   r'   r   r)   r[   ra   rb   s   @r#   r   r      s        s s g|&
 &
 &
 &
 &
 &
PuS#X        r%   r   c                        e Zd ZdZg ddddej        ej        df fd	Zdede	e
e
f         fdZdede	e
e
f         fd	Zd
ede	e
e
f         fdZdee	e
e
f                  fdZ xZS )ParallelBlockz Parallel block class. Frf   Nc           	         t                                                        |	|d                   | _         |	|d                   | _         |	|d                   | _        t          |d         |||||
d                   | _        t          |d         |||||
d                   | _        t          |d         |||||
d                   | _        |dk    rt          |          nt          j                    | _         |	|d                   | _         |	|d                   | _         |	|d                   | _        |d         |d         cxk    r|d         k    sn J |d         |d         cxk    r|d         k    sn J t!          |d         |d         z            }t#          |d         |||          x| _        x| _        | _        d S )Nr   r   rh   r   rf   r   )r&   r'   norm12norm13norm14rd   factoratt_crpe2factoratt_crpe3factoratt_crpe4r   r-   r   r   norm22norm23norm24r)   r   mlp2mlp3mlp4)r5   dimsr6   
mlp_ratiosrr   rp   rn   r   r   r   shared_crpesr   r;   s               r#   r'   zParallelBlock.__init__   s    	 !ja)) ja)) ja))6G$Q 
  
  
  7G$Q 
  
  
  7G$Q 
  
  
 1:B),,,BKMM !ja)) ja)) ja))Aw$q',,,,T!W,,,,,,!}
1>>>>A>>>>>>T!Wz!}455,/Q*	-
 -
 -
 	
	 	
DI			r%   factorr<   c                 2    |                      |||          S )z Feature map up-sampling. scale_factorr<   interpolater5   r!   r   r<   s       r#   upsamplezParallelBlock.upsample  s    TBBBr%   c                 8    |                      |d|z  |          S )z Feature map down-sampling.       ?r   r   r   s       r#   
downsamplezParallelBlock.downsample#  s!    F
FFFr%   r   c                    |j         \  }}}|\  }}t          |d||z  z   k    d           |ddddddf         }	|ddddddf         }
|
                    dd                              ||||          }
t	          j        |
|ddd          }
|
                    ||d                              dd          }
t          j        |	|
fd	          }|S )
z Feature map interpolation. r   r>   Nr   Fbilinear)r   recompute_scale_factormodealign_cornersr?   rA   )rC   r   rD   rE   rJ   r   rF   rI   )r5   r!   r   r<   rN   rO   rP   rQ   rR   r   r   outs               r#   r   zParallelBlock.interpolate'  s    '1a1QQY###aaa!QQQhK	qqq!""aaax[
))!Q//771aCC
]%#(
 
 

  ''1b11;;AqAA
iJ/Q777
r%   sizesc                    |\  }}}}	|                      |          }
|                     |          }|                     |          }|                     |
|          }
|                     ||          }|                     ||	          }|                     |d|          }|                     |d|	          }|                     |d|	          }|                     |
d|          }|                     |d|          }|                     |
d|          }|
|z   |z   }
||z   |z   }||z   |z   }||                     |
          z   }||                     |          z   }||                     |          z   }| 	                    |          }
| 
                    |          }|                     |          }|                     |
          }
|                     |          }|                     |          }||                     |
          z   }||                     |          z   }||                     |          z   }||||fS )Nrw   g       @)r   r<   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r5   x1x2x3x4r   _S2S3S4cur2cur3cur4upsample3_2upsample4_3upsample4_2downsample2_3downsample3_4downsample2_4s                      r#   r[   zParallelBlock.forward>  s)   2r2{{2{{2{{2##Dr#22##Dr#22##Dr#22mmD"m==mmD"m==mmD"m==RbAARbAARbAAk!K/k!M1m#m3$..&&&$..&&&$..&&& {{2{{2{{2yyyyyy$..&&&$..&&&$..&&&2r2~r%   )r]   r^   r_   r`   r-   r   r   r'   floatr   r)   r   r   r   r   r[   ra   rb   s   @r#   r   r      s       !!
 g|:
 :
 :
 :
 :
 :
xC% CuS#X C C C CGE GsCx G G G G5 c3h    . T%S/-B                r%   r   c                       e Zd ZdZdddddddd	d
dddddeddddf fd	Zd Zej        j	        d             Z
ej        j	        dd            Zej        j	        d d            Zej        j	        dej        fd            Zd!dedee         fdZd Zd deej        eej                 f         defdZdej        fdZ xZS )"r   z CoaT class.       rh     @      @     rh   rv      rh   r   re   )rv   rv   rv   rv   Trf   FNtokenc                 n   	 t                                                       |dv sJ |pdddd}| _        | _         _        d         x _         _        | _        | _        t          |          }t          |||d         t          j                   _        t          d |D             dd         d	         t          j                   _        t          d
 |D             dd	         d         t          j                   _        t          d |D             dd         d         t          j                   _        t          j        t%          j        d	d	d                              _        t          j        t%          j        d	d	d	                              _        t          j        t%          j        d	d	d                              _        t          j        t%          j        d	d	d                              _        t1          d         d           _        t1          d	         d           _        t1          d         d           _        t1          d         d           _        t;          d         |z  ||           _        t;          d	         |z  ||           _        t;          d         |z  ||           _         t;          d         |z  ||           _!        |}|dk    sJ tE          ||
||||          t          j#        	 fdtI          |d                   D                        _%        t          j#        	 fdtI          |d	                   D                        _&        t          j#        	 fdtI          |d                   D                        _'        t          j#        	 fdtI          |d                   D                        _(        | _)         j)        dk    r6t          j#        	 fdtI          |          D                        _*        nd  _*         j        sW j*        - |d	                    _+         |d                    _,        nd x _+         _,         |d                    _-         j)        dk    rd	         d         cxk    rd         k    sn J t$          j        .                    dd	d	           _/        t          j0        |           _1        |dk    rt          j2         j        |          nt          j3                     _4        nXd  _/        t          j0        |           _1        |dk    rt          j2         j        |          nt          j3                     _4        tk           j        d           tk           j        d           tk           j        d           tk           j        d            6                     j7                   d S )Nr   avgr   rh   )rh         r?   r   )img_size
patch_sizein_chans	embed_dimr   c                     g | ]}|d z  S )rv   r   r    r!   s     r#   r$   z!CoaT.__init__.<locals>.<listcomp>      ///a1f///r%   r   c                     g | ]}|d z  S )re   r   r   s     r#   r$   z!CoaT.__init__.<locals>.<listcomp>  r   r%   c                     g | ]}|d z  S )r   r   r   s     r#   r$   z!CoaT.__init__.<locals>.<listcomp>  s    000!a2g000r%   )rB   r{   )r"   r6   r*   rf   )r6   rr   rp   rn   r   r   c           
      ^    g | ])}t          dd          d          j        j        d*S )r   rB   r   r   rs   r   )r   cpe1crpe1r    r   
embed_dimsr   r5   skwargss     r#   r$   z!CoaT.__init__.<locals>.<listcomp>  a     -. -. -.   qM$Q-9 J	 
  -. -. -.r%   c           
      ^    g | ])}t          dd          d          j        j        d*S )r   r   r   )r   cpe2crpe2r   s     r#   r$   z!CoaT.__init__.<locals>.<listcomp>  r   r%   c           
      ^    g | ])}t          dd          d          j        j        d*S )r   r   r   )r   cpe3crpe3r   s     r#   r$   z!CoaT.__init__.<locals>.<listcomp>  r   r%   c           
      ^    g | ])}t          dd          d          j        j        d*S )rh   r   r   )r   cpe4crpe4r   s     r#   r$   z!CoaT.__init__.<locals>.<listcomp>  r   r%   c                 `    g | ]*}t          dj        j        j        j        fd +S ))r   r   r   r   )r   r   r   r   r  r   s     r#   r$   z!CoaT.__init__.<locals>.<listcomp>  sc     20 20 20   #)"&*dj$*dj!Q  	 20 20 20r%   )in_channelsout_channelsr   {Gz?std)8r&   r'   return_interm_layersout_featuresr   num_featureshead_hidden_sizenum_classesglobal_poolr   r
   r-   r   patch_embed1patch_embed2patch_embed3patch_embed4	ParameterrF   zeros
cls_token1
cls_token2
cls_token3
cls_token4r   r   r   r   r   r   r   r   r   r  r+   r.   rangeserial_blocks1serial_blocks2serial_blocks3serial_blocks4parallel_depthparallel_blocksr   norm3norm4Conv1d	aggregaterm   	head_droprk   r   headr   apply_init_weights)r5   r   r   r   r  r   serial_depthsr  r6   r   rr   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   r  r	  crpe_windowr  dprr   r;   s   `    `   `           @r#   r'   zCoaT.__init__c  s   , 	.....!7qQ%7%7$8!($4>rNBD1&& X&&&*x m> > > '//h///A
ST m> > > '//h///A
ST m> > > '00x000QTU m> > >
 ,u{1aA'G'GHH,u{1aA'G'GHH,u{1aA'G'GHH,u{1aA'G'GHH :a=A666	:a=A666	:a=A666	:a=A666	 #JqMY,FR[doppp
"JqMY,FR[doppp
"JqMY,FR[doppp
"JqMY,FR[doppp
 czzzz$$!
 
 
 !m -. -. -. -. -. -. -. =+,,-. -. -. 	
 	
 !m -. -. -. -. -. -. -. =+,,-. -. -. 	
 	
 !m -. -. -. -. -. -. -. =+,,-. -. -. 	
 	
 !m -. -. -. -. -. -. -. =+,,-. -. -. 	
 	
 -""#%= 20 20 20 20 20 20 20 ~..20 20 20 $ $D   $(D  ( 	l#/'Z
166
'Z
166

*..
TZ#JqM22DJ"Q&&!!}
1FFFFAFFFFFF!&QQ\]!^!^!#I!6!6ITWXBId&7EEE^`^i^k^k		 "&!#I!6!6ITWXBId&7EEE^`^i^k^k	 	do3////do3////do3////do3////

4%&&&&&r%   c                    t          |t          j                  rbt          |j        d           t          |t          j                  r.|j        )t          j                            |j        d           d S d S d S t          |t          j                  rLt          j                            |j        d           t          j                            |j        d           d S d S )Nr  r  r   r   )	r(   r-   rk   r   weightri   init	constant_r   )r5   ms     r#   r&  zCoaT._init_weights  s    a## 	-!(,,,,!RY'' -AF,>!!!&!,,,,,- -,>,>2<(( 	-Gafa(((Gah,,,,,	- 	-r%   c                 
    h dS )N>   r  r  r  r  r   r5   s    r#   no_weight_decayzCoaT.no_weight_decay  s    GGGGr%   c                     |r
J d            d S )Nz$gradient checkpointing not supportedr   )r5   enables     r#   set_grad_checkpointingzCoaT.set_grad_checkpointing  s    AAAAAAAAr%   c                 :    t          ddddddddd	d
g	  	        }|S )Nz#^cls_token1|patch_embed1|crpe1|cpe1z^serial_blocks1\.(\d+)z#^cls_token2|patch_embed2|crpe2|cpe2z^serial_blocks2\.(\d+)z#^cls_token3|patch_embed3|crpe3|cpe3z^serial_blocks3\.(\d+)z#^cls_token4|patch_embed4|crpe4|cpe4z^serial_blocks4\.(\d+))z^parallel_blocks\.(\d+)N)z^norm|aggregate)i )	stem1r  stem2r  stem3r  stem4r  r  )r+   )r5   coarsematchers      r#   group_matcherzCoaT.group_matcher  s@    848484842.
 
 
 r%   returnc                     | j         S r   )r$  r4  s    r#   get_classifierzCoaT.get_classifier.  s
    yr%   r  r  c                     || _         ||dv sJ || _        |dk    rt          j        | j        |          nt          j                    | _        d S )Nr   r   )r  r  r-   rk   r
  r   r$  )r5   r  r  s      r#   reset_classifierzCoaT.reset_classifier2  s[    &""22222*DALqBId/===VXVaVcVc			r%   c                 
   |j         d         }|                     |          }| j        j        \  }}t          || j                  }| j        D ]} ||||f          }t          |                              |||d                              dddd          	                                }| 
                    |          }| j
        j        \  }	}
t          || j                  }| j        D ]} |||	|
f          }t          |                              ||	|
d                              dddd          	                                }|                     |          }| j        j        \  }}t          || j                  }| j        D ]} ||||f          }t          |                              |||d                              dddd          	                                }|                     |          }| j        j        \  }}t          || j                  }| j        D ]} ||||f          }t          |                              |||d                              dddd          	                                }| j        xt(          j                                        sC| j        r<i }d| j        v r||d<   d| j        v r||d<   d	| j        v r||d	<   d
| j        v r||d
<   |S |                     |          }|S | j        D ]k}|                     ||	|
f          |                     |||f          |                     |||f          }}} |||||||f|	|
f||f||fg          \  }}}}lt(          j                                        sx| j        rpi }d| j        v rRt          |                              |||d                              dddd          	                                }||d<   d| j        v rRt          |                              ||	|
d                              dddd          	                                }||d<   d	| j        v rRt          |                              |||d                              dddd          	                                }||d	<   d
| j        v rRt          |                              |||d                              dddd          	                                }||d
<   |S |                     |          }|                     |          }|                     |          }|||gS )Nr   rw   r?   rh   r   r   x1_noclsx2_noclsx3_noclsx4_nocls)r   )rC   r  	grid_size
insert_clsr  r  
remove_clsrE   rx   
contiguousr  r  r  r  r  r  r  r  r  r  rF   jitis_scriptingr  r	  r   r   r   r   r   r  )r5   x0rN   r   H1W1blkrG  r   H2W2rH  r   H3W3rI  r   H4W4rJ  feat_outs                        r#   forward_featureszCoaT.forward_features9  s   HQK r""",BDO,,& 	( 	(CRr2h'''BBb>>))!RR88@@Aq!LLWWYY x((",BDO,,& 	( 	(CRr2h'''BBb>>))!RR88@@Aq!LLWWYY x((",BDO,,& 	( 	(CRr2h'''BBb>>))!RR88@@Aq!LLWWYY x((",BDO,,& 	( 	(CRr2h'''BBb>>))!RR88@@Aq!LLWWYY '9))++ 0I !222+3HZ(!222+3HZ(!222+3HZ(!222+3HZ( ZZ^^	 ' 	a 	aC2Bx00$))BR2I2I499UWZ\^`YaKbKbBB SRRR2r(RQSHWY[]V^7_```NBBy%%'' 	 D,E 	 HT...%b>>11!RR@@HHAqRSTT__aa'/$T...%b>>11!RR@@HHAqRSTT__aa'/$T...%b>>11!RR@@HHAqRSTT__aa'/$T...%b>>11!RR@@HHAqRSTT__aa'/$OBBBBBBB<r%   x_feat
pre_logitsc                    t          |t                    r| j        J | j        dk    r!t	          j        d |D             d          }n t	          j        d |D             d          }|                     |                              d          }n9| j        dk    r"|d d dd f                             d          n|d d df         }| 	                    |          }|r|n| 
                    |          S )Nr   c                 T    g | ]%}|d d dd f                              dd          &S )Nr   T)rB   keepdim)meanr    xls     r#   r$   z%CoaT.forward_head.<locals>.<listcomp>  s8    TTTrr!!!QRR%y~~!T~BBTTTr%   r   rA   c                 &    g | ]}|d d df         S )Nr   r   rc  s     r#   r$   z%CoaT.forward_head.<locals>.<listcomp>  s$     ; ; ;bAAAqD ; ; ;r%   r   )r(   listr"  r  rF   rI   stacksqueezerb  r#  r$  )r5   r]  r^  r!   s       r#   forward_headzCoaT.forward_head  s   fd## 	Y>---5((ITTVTTTZ[\\\K ; ;F ; ; ;CCCq!!))a)00AA-1-=-F-Fqqq!""u""q")))FSTSTSTVWSWLANN10qqDIIaLL0r%   c                     t           j                                        s| j        r|                     |          S |                     |          }|                     |          }|S r   )rF   rO  rP  r  r\  ri  )r5   r!   r]  s      r#   r[   zCoaT.forward  sc    y%%'' 	D,E 	((+++ **1--F!!&))AHr%   )TFr   )r]   r^   r_   r`   r   r'   r&  rF   rO  ignorer5  r8  r@  r-   ModulerC  r)   r   strrE  r\  r   Tensorr   boolri  r[   ra   rb   s   @r#   r   r   a  s        *&# !&)f' f' f' f' f' f'P- - - YH H H YB B B B Y   " Y	    d dC dhsm d d d dO  O  O b1 15tEL7I)I#J 1X\ 1 1 1 1EL        r%   c                 z    |                     | j        d         dd          }t          j        || fd          } | S )z Insert CLS token. r   r?   r   rA   )expandrC   rF   rI   )r!   r   
cls_tokenss      r#   rL  rL    s<    !!!'!*b"55J	:q/q)))AHr%   c                 $    | ddddddf         S )z Remove CLS token. Nr   r   )r!   s    r#   rM  rM    s    QQQAAAX;r%   c                    i }|                      d|           } |                                 D ]\  }}|                    d          s|                    d          rt          |dd           |                    d          rt          |dd           r|                    d          rt          |dd           L|                    d          rt          |dd           &|                    d          rt          |dd           |||<   |S )Nmodelr   r   r  r   r"  r$  )getr1   
startswithgetattr)
state_dictrv  out_dictr{   rM   s        r#   checkpoint_filter_fnr|    s"   H44J  "" 	 	1<<   	g&&	+25'4+H+H+Pg&& ,Q+25'4+H+H+Pg&& ,Q+25'4+H+H+Pk** ,Q/6uk4/P/P/Xf%% 0Y*1%*F*F*NOr%   Fc                     |                     dd           rt          d          t          t          | |fdt          i|}|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.pretrained_filter_fn)rw  RuntimeErrorr   r   r|  )variant
pretraineddefault_cfgkwargsrv  s        r#   _create_coatr    s`    zz/4(( [YZZZ   2	
  E Lr%   r>   c                 6    | ddd dddt           t          ddd|S )	Nr   )rh   r   r   g?bicubicTzpatch_embed1.projr$  )urlr  
input_size	pool_sizecrop_pctinterpolationfixed_input_sizerb  r  
first_conv
classifierr   )r  r  s     r#   	_cfg_coatr    s7    =t%.B)   r%   ztimm/)	hf_hub_id)rh     r  r   squash)r  r  r  	crop_mode)zcoat_tiny.in1kzcoat_mini.in1kzcoat_small.in1kzcoat_lite_tiny.in1kzcoat_lite_mini.in1kzcoat_lite_small.in1kzcoat_lite_medium.in1kzcoat_lite_medium_384.in1krA  c           	      f    t          dg dg dd          }t          dd| it          |fi |}|S )	Nrv   )   r  r  r  r   r   r   r   r   r   r   r'  r  	coat_tinyr  )r  r+   r  r  r  	model_cfgrv  s       r#   r  r    \    !5!5!5\\\bce e eIYYYtI?X?XQW?X?XYYELr%   c           	      f    t          dg dg dd          }t          dd| it          |fi |}|S )	Nrv   )r     r  r  r  r   r  	coat_minir  )r  r  r  s       r#   r  r    r  r%   c           	      b    t          ddg dg ddd|}t          d	d| it          |fi |}|S )
Nrv   )r  r   r   r   r  r   r  
coat_smallr  r   )r  r  r  s       r#   r  r    sk     o!5!5!5\\\bco ogmo oIZZ*ZY@Y@YRX@Y@YZZELr%   c           	      j    t          dg dg dg d          }t          dd| it          |fi |}|S )	Nrv   )r   r      r   r  re   re   rv   rv   r   r   r'  r   coat_lite_tinyr  )r  r  r  s       r#   r  r    b    !4!4!4LLL]i]i]ik k kI^^j^DD]D]V\D]D]^^ELr%   c           	      j    t          dg dg dg d          }t          dd| it          |fi |}|S )	Nrv   r   r  r  r  coat_lite_minir  )r  r  r  s       r#   r  r    r  r%   c           	      j    t          dg dg dg d          }t          dd| it          |fi |}|S )	Nrv   r   r   r  r  coat_lite_smallr  )r  r  r  s       r#   r  r    sb    !4!4!4LLL]i]i]ik k kI__z_T)E^E^W]E^E^__ELr%   c           	      d    t          dg dg d          }t          dd| it          |fi |}|S )Nrv   r   r  r   r   rh   r   
   re   )r   r   r'  coat_lite_mediumr  )r  r  r  s       r#   r  r    sY    !5!5!5]]]T T TI``
`d9F_F_X^F_F_``ELr%   c           	      f    t          ddg dg d          }t          dd| it          |fi |}|S )	Nr  rv   r  r  )r   r   r   r'  coat_lite_medium_384r  )r  r  r  s       r#   r  r    s^    /C/C/CS`S`S`b b bIddJd$yJcJc\bJcJcddELr%   )FN)r>   rk  )2r`   typingr   r   r   r   rF   torch.nnr-   torch.nn.functional
functionalrJ   	timm.datar   r	   timm.layersr
   r   r   r   r   r   r   _builderr   	_registryr   r   __all__rm  r   rd   r   r   r   r   rL  rM  r|  r  r  default_cfgsr  r  r  r  r  r  r  r  r   r%   r#   <module>r     sr    0 / / / / / / / / / / /                 A A A A A A A A _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ * * * * * * < < < < < < < <(? ? ? ? ?BI ? ? ?D/ / / / /bi / / /d       67 7 7 7 7") 7 7 7t} } } } }BI } } }@~ ~ ~ ~ ~29 ~ ~ ~B
    
          %$i'222i'222 y7333$9w777$9w777%I888&Y999!* 3(" " "& &    T      T      d      $      $      4      D            r%   