
    NgS                        d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ g dZ G d de
j                  Z G d de
j                  Z  G d de
j                  Z! G d de
j                  Z" G d de
j                  Z#d7dZ$d8dZ%d9dZ& e e&ddd           e&dd !           e&dd"d           e&dd#!           e&dd$!           e&dd%d           e&dd&!           e&dd'!           e&dd(!           e&dd)d*          d+
          Z'ed8d,e#fd-            Z(ed8d,e#fd.            Z)ed8d,e#fd/            Z*ed8d,e#fd0            Z+ed8d,e#fd1            Z,ed8d,e#fd2            Z-ed8d,e#fd3            Z.ed8d,e#fd4            Z/ed8d,e#fd5            Z0ed8d,e#fd6            Z1dS ):a3   Class-Attention in Image Transformers (CaiT)

Paper: 'Going deeper with Image Transformers' - https://arxiv.org/abs/2103.17239

Original code and weights from https://github.com/facebookresearch/deit, copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )partial)ListOptionalTupleUnionNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPathtrunc_normal_use_fused_attn   )build_model_with_cfg)feature_take_indices)checkpoint_seq)register_modelgenerate_default_cfgs)Cait	ClassAttnLayerScaleBlockClassAttnLayerScaleBlockTalkingHeadAttnc                   R     e Zd ZU ej        j        e         ed<   d fd	Zd Z	 xZ
S )r   
fused_attn   F        c                    t                                                       || _        ||z  }|dz  | _        t	                      | _        t          j        |||          | _        t          j        |||          | _	        t          j        |||          | _
        t          j        |          | _        t          j        ||          | _        t          j        |          | _        d S )N      ࿩bias)super__init__	num_headsscaler   r   nnLinearqkvDropout	attn_dropproj	proj_dropselfdimr%   qkv_biasr-   r/   head_dim	__class__s          L/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/cait.pyr$   zClassAttn.__init__    s    ")#%
(**3(3333(3333(333I..Ic3''	I..    c                     |j         \  }}}|                     |d d df                                       d                              |d| j        || j        z                                dddd          }|                     |                              ||| j        || j        z                                dddd          }|                     |                              ||| j        || j        z                                dddd          }| j        r<t          j
        j                            |||| j        r| j        j        nd          }nS|| j        z  }||                    dd          z  }	|	                    d	          }	|                     |	          }	|	|z  }|                    dd                              |d|          }|                     |          }|                     |          }|S )
Nr   r         r   )	dropout_pr2   )shaper)   	unsqueezereshaper%   permuter*   r+   r   torchr'   
functionalscaled_dot_product_attentiontrainingr-   pr&   	transposesoftmaxr.   r/   )
r1   xBNCr)   r*   r+   x_clsattns
             r6   forwardzClassAttn.forward.   s   '1aFF1QQQT7OO%%a((00At~qDNGZ[[ccdeghjkmnooFF1IIaDNA4GHHPPQRTUWXZ[\\FF1IIaDNA4GHHPPQRTUWXZ[\\? 
	H'DD1a.2mC$.** E  EE
 DJAq{{2r***D<<B<''D>>$''D1HE1%%--aA66		%  u%%r7   r   Fr   r   )__name__
__module____qualname__rC   jitFinalbool__annotations__r$   rP   __classcell__r5   s   @r6   r   r      se          	%%%%/ / / / / /      r7   r   c            
       N     e Zd Zdddddej        ej        eedf
 fd	Zd Z	 xZ
S )r         @Fr   -C6?c                    t                                                        |	|          | _         |
|||||          | _        |dk    rt	          |          nt          j                    | _         |	|          | _        t          ||z            } |||||          | _
        t          j        |t          j        |          z            | _        t          j        |t          j        |          z            | _        d S N)r%   r3   r-   r/   r   )in_featureshidden_features	act_layerdropr#   r$   norm1rO   r   r'   Identity	drop_pathnorm2intmlp	ParameterrC   onesgamma_1gamma_2r1   r2   r%   	mlp_ratior3   r/   r-   rg   rb   
norm_layer
attn_block	mlp_blockinit_valuesmlp_hidden_dimr5   s                 r6   r$   z!LayerScaleBlockClassAttn.__init__J        	Z__
J
 
 
	 1:B),,,BKMMZ__
S9_--9*	
 
 
 |K%*S//$ABB|K%*S//$ABBr7   c           	      N   t          j        ||fd          }||                     | j        |                     |                     |                    z            z   }||                     | j        |                     |                     |                    z            z   }|S )Nr   r>   )	rC   catrg   rm   rO   re   rn   rj   rh   )r1   rJ   rN   us       r6   rP   z LayerScaleBlockClassAttn.forwardn   s    Iuaja(((t|dii

16N6N'NOOOt|dhhtzz%?P?P6Q6Q'QRRRr7   )rR   rS   rT   r'   GELU	LayerNormr   r   r$   rP   rY   rZ   s   @r6   r   r   G   sw         g| "C "C "C "C "C "CH      r7   r   c                   &     e Zd Zd fd	Zd Z xZS )r   r   Fr   c                    t                                                       || _        ||z  }|dz  | _        t	          j        ||dz  |          | _        t	          j        |          | _        t	          j        ||          | _	        t	          j        ||          | _
        t	          j        ||          | _        t	          j        |          | _        d S )Nr    r:   r!   )r#   r$   r%   r&   r'   r(   qkvr,   r-   r.   proj_lproj_wr/   r0   s          r6   r$   zTalkingHeadAttn.__init__x   s    ")#%
9S#'999I..Ic3''	i	955i	955I..r7   c                 8   |j         \  }}}|                     |                              ||d| j        || j        z                                ddddd          }|d         | j        z  |d         |d         }}}||                    dd          z  }	|                     |	                    dddd                                        dddd          }	|	                    d          }	| 	                    |	                    dddd                                        dddd          }	| 
                    |	          }	|	|z                      dd                              |||          }|                     |          }|                     |          }|S )	Nr:   r9   r   r      r<   r=   r>   )r?   r~   rA   r%   rB   r&   rH   r   rI   r   r-   r.   r/   )
r1   rJ   rK   rL   rM   r~   r)   r*   r+   rO   s
             r6   rP   zTalkingHeadAttn.forward   sr   '1ahhqkk!!!Q4>1;NOOWWXY[\^_abdeffa&4:%s1vs1va11;;r2&&&{{4<<1a3344<<Q1aHH|||##{{4<<1a3344<<Q1aHH~~d##AX  A&&..q!Q77IIaLLNN1r7   rQ   )rR   rS   rT   r$   rP   rY   rZ   s   @r6   r   r   u   sL        / / / / / /&      r7   r   c            
       N     e Zd Zdddddej        ej        eedf
 fd	Zd Z	 xZ
S )r   r\   Fr   r]   c                    t                                                        |	|          | _         |
|||||          | _        |dk    rt	          |          nt          j                    | _         |	|          | _        t          ||z            } |||||          | _
        t          j        |t          j        |          z            | _        t          j        |t          j        |          z            | _        d S r_   rd   ro   s                 r6   r$   zLayerScaleBlock.__init__   rv   r7   c           	         ||                      | j        |                     |                     |                    z            z   }||                      | j        |                     |                     |                    z            z   }|S N)rg   rm   rO   re   rn   rj   rh   r1   rJ   s     r6   rP   zLayerScaleBlock.forward   sm    t|dii

1.F.FFGGGt|dhhtzz!}}.E.EEFFFr7   )rR   rS   rT   r'   rz   r{   r   r   r$   rP   rY   rZ   s   @r6   r   r      sw         g|&"C "C "C "C "C "CH      r7   r   c                   j    e Zd Zdddddddddd	d
d
d
d
d
eee eej        d          ej	        e
edeeddf fd	Zd Zej        j        d             Zej        j        d+d            Zej        j        d,d            Zej        j        dej        fd            Zd-dedee         fdZ	 	 	 	 	 d.dej        deeeee         f                  deded ed!edeeej                 eej        eej                 f         f         fd"Z 	 	 	 d/deeee         f         d$ed%efd&Z!d' Z"d,d(efd)Z#d* Z$ xZ%S )0r         r:     token      r\   Tr   ư>)epsr]   r9   c                   	
 t                                                       |dv sJ || _        || _        x| _        x| _        | _        d| _         ||||          | _        | j        j	        }t          | j        d          r| j                                        n|t          j        t          j        dd                    | _        t          j        t          j        d|                    | _        t          j        |          | _        fdt)          |          D             t          j        	
fdt)          |          D              | _        fd	t)          |          D             | _        t          j        
f
d
t)          |          D                       | _                   | _        t          j        |          | _        |dk    rt          j        |          nt          j                    | _        t?          | j        d           t?          | j        d           |                      | j!                   d S )N r   avgF)img_size
patch_sizein_chans	embed_dim
feat_ratior   )rG   c                     g | ]}S  r   ).0idrop_path_rates     r6   
<listcomp>z!Cait.__init__.<locals>.<listcomp>  s    444!~444r7   c                 J    g | ]} 
|         	            S ))r2   r%   rp   r3   r/   r-   rg   rq   rb   rr   rs   rt   r   )r   r   rb   rr   attn_drop_rateblock_layersdprr   rt   rs   rp   rq   r%   proj_drop_rater3   s     r6   r   z!Cait.__init__.<locals>.<listcomp>  sd     &! &! &!  '3l$$!f!!#'
 '
 '
 &! &! &!r7   c                 :    g | ]}t          d |           S )blocks.)num_chs	reductionmodule)dict)r   r   r   rs     r6   r   z!Cait.__init__.<locals>.<listcomp>  s0    nnn\]T)qSTWWWnnnr7   c                 8   
 g | ]} 
	 	  	        S ))	r2   r%   rp   r3   rq   rb   rr   rs   rt   r   )r   _rb   attn_block_token_onlyblock_layers_tokenr   rt   mlp_block_token_onlymlp_ratio_token_onlyrq   r%   r3   s     r6   r   z!Cait.__init__.<locals>.<listcomp>  sY     
0, 
0, 
0,  1C0B*!,*#
1
 
1
 
1
 
0, 
0, 
0,r7   r   {Gz?std)"r#   r$   num_classesglobal_poolnum_featureshead_hidden_sizer   grad_checkpointingpatch_embednum_patcheshasattrr   r'   rk   rC   zeros	cls_token	pos_embedr,   pos_droprange
Sequentialblocksfeature_info
ModuleListblocks_token_onlynorm	head_dropr(   rf   headr   apply_init_weights) r1   r   r   r   r   r   r   depthr%   rp   r3   	drop_ratepos_drop_rater   r   r   r   r   patch_layerrq   rb   rr   rs   rt   r   r   depth_token_onlyr   r   r   r   r5   s          ` ```  ````` ``````` ` @@r6   r$   zCait.__init__   s   < 	22222&&ENNND1DN"'&;!	
 
 
 &2-4T5E|-T-TdD'')))Zdek!Q	&B&BCCek![)&L&LMM
]3334444uU||444m &! &! &! &! &! &! &! &! &! &! &! &! &! &! &! &! <<&! &! &! " onnnnafglamamnnn!# 
0, 
0, 
0, 
0, 
0, 
0, 
0, 
0, 
0, 
0, 
0, 
0, 
0, )**
0, 
0, 
0, 
"- 
"- Jy))	I..9DqBIi555bkmm	dn#....dn#....

4%&&&&&r7   c                    t          |t          j                  rbt          |j        d           t          |t          j                  r.|j        )t          j                            |j        d           d S d S d S t          |t          j                  rLt          j                            |j        d           t          j                            |j        d           d S d S )Nr   r   r         ?)	
isinstancer'   r(   r   weightr"   init	constant_r{   )r1   ms     r6   r   zCait._init_weights(  s    a## 	-!(,,,,!RY'' -AF,>!!!&!,,,,,- -,>,>2<(( 	-Gafa(((Gah,,,,,	- 	-r7   c                 
    ddhS )Nr   r   r   r1   s    r6   no_weight_decayzCait.no_weight_decay1  s    [))r7   c                     || _         d S r   )r   )r1   enables     r6   set_grad_checkpointingzCait.set_grad_checkpointing5  s    "(r7   Fc                       fd}|S )Nc                     t           fddD                       rdS                      d          r+t                               d          d                   dz   S                      d          rWt	          j                  t	          j                  z
  dz   }t                               d          d                   |z   S                      d          rt	          j                  S t          d	          S )
Nc                 :    g | ]}                     |          S r   )
startswith)r   nnames     r6   r   z8Cait.group_matcher.<locals>._matcher.<locals>.<listcomp><  s%    ZZZ1DOOA&&ZZZr7   )r   r   r   r   r   .r   zblocks_token_only.znorm.inf)anyr   ri   splitlenr   r   float)r   	to_offsetr1   s   ` r6   _matcherz$Cait.group_matcher.<locals>._matcher;  s    ZZZZ0YZZZ[[ $q++ 	$4::c??1-..22!566 $,,s43I/J/JJQN	4::c??1-..::)) $4;'''U||#r7   r   )r1   coarser   s   `  r6   group_matcherzCait.group_matcher9  s#    	$ 	$ 	$ 	$ 	$ r7   returnc                     | j         S r   )r   r   s    r6   get_classifierzCait.get_classifierJ  s
    yr7   Nr   r   c                     || _         ||dv sJ || _        |dk    rt          j        | j        |          nt          j                    | _        d S )Nr   r   )r   r   r'   r(   r   rf   r   )r1   r   r   s      r6   reset_classifierzCait.reset_classifierN  s[    &""66666*DALqBId/===VXVaVcVc			r7   NCHWrJ   indicesr   
stop_early
output_fmtintermediates_onlyc                 x   |dv s
J d            |dk    }g }t          t          | j                  |          \  }	}
|j        \  }}}|                     |          }|| j        z   }|                     |          }t          j        	                                s|s| j        }n| j        d|
dz            }t          |          D ]@\  }} ||          }||	v r,|                    |r|                     |          n|           A|r/| j                            ||f          \  fd|D             }|r|S | j                            |j        d         dd          }t          | j                  D ]\  }} |||          }t          j        ||fd	          }|                     |          }||fS )
a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        )r   NLCz)Output format must be one of NCHW or NLC.r   Nr   c                     g | ]B}|                     d                               dddd                                          CS )r=   r   r:   r   r9   )rA   rB   
contiguous)r   yrK   HWs     r6   r   z.Cait.forward_intermediates.<locals>.<listcomp>  sL    lllYZQYYq!Q33;;Aq!QGGRRTTlllr7   r   r=   r>   )r   r   r   r?   r   r   r   rC   rU   is_scripting	enumerateappendr   dynamic_feat_sizer   expandr   rx   )r1   rJ   r   r   r   r   r   rA   intermediatestake_indices	max_indexr   heightwidthr   r   blk
cls_tokensrK   r   r   s                     @@@r6   forward_intermediateszCait.forward_intermediatesU  s   & _,,,.Y,,,&"6s4;7G7G"Q"Qi  g1feQMM!9!!## 	1: 	1[FF[)a-0F'' 	B 	BFAsAAL  $$T%@TYYq\\\qAAA  	m#55vuoFFDAqllllll^klllM 	!   ^**171:r2>>
 677 	, 	,FAsQ
++JJIz1o1---IIaLL-r7   r   
prune_norm
prune_headc                    t          t          | j                  |          \  }}| j        d|dz            | _        |rt          j                    | _        |r.t          j                    | _        |                     dd           |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )	r   r   r   r'   rf   r   r   r   r   )r1   r   r  r  r  r  s         r6   prune_intermediate_layerszCait.prune_intermediate_layers  s     #7s4;7G7G"Q"Qik.9q=.1 	&DI 	)%']__D"!!!R(((r7   c                    |                      |          }|| j        z   }|                     |          }| j        r4t          j                                        st          | j        |          }n|                     |          }| j	        
                    |j        d         dd          }t          | j                  D ]\  }} |||          }t	          j        ||fd          }|                     |          }|S )Nr   r=   r   r>   )r   r   r   r   rC   rU   r   r   r   r   r  r?   r   r   rx   r   )r1   rJ   r
  r   r	  s        r6   forward_featureszCait.forward_features  s    QMM!" 	59+A+A+C+C 	t{A..AAAA^**171:r2>>
 677 	, 	,FAsQ
++JJIz1o1---IIaLLr7   
pre_logitsc                     | j         r9| j         dk    r"|d d dd f                             d          n|d d df         }|                     |          }|r|n|                     |          S )Nr   r   r>   r   )r   meanr   r   )r1   rJ   r  s      r6   forward_headzCait.forward_head  sy     	O(,(8E(A(A!!!QRR%!$$$qAwANN10qqDIIaLL0r7   c                 Z    |                      |          }|                     |          }|S r   )r  r  r   s     r6   rP   zCait.forward  s-    !!!$$a  r7   )TFr   )NFFr   F)r   FT)&rR   rS   rT   r   r   r   r   r'   r{   rz   r   r   r   r$   r   rC   rU   ignorer   r   r   Moduler   ri   r   strr   Tensorr   r   rW   r   r  r  r  r  rP   rY   rZ   s   @r6   r   r      s       
 (7"wr|666g&"+!$!$9W' W' W' W' W' W'r- - - Y* * * Y) ) ) ) Y     Y	    d dC dhsm d d d d 8<$$',8  8 |8  eCcN348  	8 
 8  8  !%8  
tEL!5tEL7I)I#JJ	K8  8  8  8 x ./$#	 3S	>*  	   "  1 1$ 1 1 1 1      r7   r   c                     d| v r| d         } i }|                                  D ]\  }}|||                    dd          <   |S )Nmodelzmodule.r   )itemsreplace)
state_dictr  checkpoint_no_moduler*   r+   s        r6   checkpoint_filter_fnr"    s^    *(
  "" ; ;19:QYYy"5566r7   Fc                     |                     dd          }t          t          | |ft          t	          |d          d|}|S )Nout_indicesr:   getter)r$  feature_cls)pretrained_filter_fnfeature_cfg)popr   r   r"  r   )variant
pretrainedkwargsr$  r  s        r6   _create_caitr-    s]    **]A..K  2[hGGG   E Lr7   r   c                 6    | ddd dddt           t          ddd|S )	Nr   )r:     r/  r   bicubicTzpatch_embed.projr   )urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer  r   
first_conv
classifierr   )r1  r,  s     r6   _cfgr9    s7    =t)%.B(   r7   ztimm/z1https://dl.fbaipublicfiles.com/deit/XXS24_224.pth)r:   r   r   )	hf_hub_idr1  r2  z1https://dl.fbaipublicfiles.com/deit/XXS24_384.pth)r:  r1  z1https://dl.fbaipublicfiles.com/deit/XXS36_224.pthz1https://dl.fbaipublicfiles.com/deit/XXS36_384.pthz0https://dl.fbaipublicfiles.com/deit/XS24_384.pthz/https://dl.fbaipublicfiles.com/deit/S24_224.pthz/https://dl.fbaipublicfiles.com/deit/S24_384.pthz/https://dl.fbaipublicfiles.com/deit/S36_384.pthz/https://dl.fbaipublicfiles.com/deit/M36_384.pthz/https://dl.fbaipublicfiles.com/deit/M48_448.pth)r:     r;  )
zcait_xxs24_224.fb_dist_in1kzcait_xxs24_384.fb_dist_in1kzcait_xxs36_224.fb_dist_in1kzcait_xxs36_384.fb_dist_in1kzcait_xs24_384.fb_dist_in1kzcait_s24_224.fb_dist_in1kzcait_s24_384.fb_dist_in1kzcait_s36_384.fb_dist_in1kzcait_m36_384.fb_dist_in1kzcait_m48_448.fb_dist_in1kr   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr         r   h㈵>r   r   r   r%   rt   cait_xxs24_224r+  )rA  r   r-  r+  r,  
model_argsr  s       r6   rA  rA    I    s"W[\\\J__j_DD^D^W]D^D^__ELr7   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr   r=  r>  r   r?  r@  cait_xxs24_384r+  )rG  rB  rC  s       r6   rG  rG    rE  r7   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr   r=  $   r   r?  r@  cait_xxs36_224r+  )rJ  rB  rC  s       r6   rJ  rJ    rE  r7   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr   r=  rI  r   r?  r@  cait_xxs36_384r+  )rL  rB  rC  s       r6   rL  rL  !  rE  r7   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr   i   r>     r?  r@  cait_xs24_384r+  )rO  rB  rC  s       r6   rO  rO  (  sI    s"W[\\\J^^Z^4
C]C]V\C]C]^^ELr7   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr   r/  r>  r   r?  r@  cait_s24_224r+  )rQ  rB  rC  s       r6   rQ  rQ  /  I    s"W[\\\J]]J]$zB\B\U[B\B\]]ELr7   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr   r/  r>  r   r?  r@  cait_s24_384r+  )rT  rB  rC  s       r6   rT  rT  6  rR  r7   c           	      `    t          ddddd          }t          d	d| it          |fi |}|S )
Nr   r/  rI  r   r   r@  cait_s36_384r+  )rV  rB  rC  s       r6   rV  rV  =  rR  r7   c           	      `    t          ddddd          }t          dd| it          |fi |}|S )	Nr   r   rI  r   r@  cait_m36_384r+  )rX  rB  rC  s       r6   rX  rX  D  I    s"X\]]]J]]J]$zB\B\U[B\B\]]ELr7   c           	      `    t          ddddd          }t          dd| it          |fi |}|S )	Nr   r   0   r   r@  cait_m48_448r+  )r\  rB  rC  s       r6   r\  r\  K  rY  r7   r   r  )r   )2__doc__	functoolsr   typingr   r   r   r   rC   torch.nnr'   	timm.datar	   r
   timm.layersr   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r  r   r   r   r   r   r"  r-  r9  default_cfgsrA  rG  rJ  rL  rO  rQ  rT  rV  rX  r\  r   r7   r6   <module>ri     s&          / / / / / / / / / / / /        A A A A A A A A P P P P P P P P P P P P P P * * * * * * + + + + + + ' ' ' ' ' ' < < < < < < < <
a
a
a) ) ) ) )	 ) ) )X+ + + + +ry + + +\' ' ' ' 'bi ' ' 'T* * * * *bi * * *Zl l l l l29 l l l^       
 
 
 
    %$#'4? $ $ $
 $(4?$ $ $ $(4? $ $ $
 $(4?$ $ $ #'$># # # "&= " " "
 "&=" " " "&=" " " "&=" " " "&= " " "Q-& -& - -`  $      $      $      $                                          r7   