
    NgC                        d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	Z	ddl
mZ ddlmZmZmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ dgZd-dej        dede fdZ! G d dej                  Z" G d dej                  Z# G d de#          Z$ G d dej                  Z% G d dej                  Z& G d dej                  Z' G d d ej                  Z( G d! dej                  Z)d" Z*d.d#Z+d/d%Z, e e,d&'           e,d&'           e,d&'          d(          Z-ed.d)e)fd*            Z.ed.d)e)fd+            Z/ed.d)e)fd,            Z0dS )0zk Sequencer

Paper: `Sequencer: Deep LSTM for Image Classification` - https://arxiv.org/pdf/2205.01972.pdf

    N)partial)
accumulate)OptionalTuple)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDDEFAULT_CROP_PCT)lecun_normal_DropPathMlp
PatchEmbedClassifierHead   )build_model_with_cfg)named_apply)register_modelgenerate_default_cfgsSequencer2d        Fmodulename	head_biasc                    t          | t          j                  r$|                    d          rKt          j                            | j                   t          j                            | j        |           d S |rCt          | j                   | j        &t          j                            | j                   d S d S t          j        
                    | j                   | j        Rd|v r(t          j                            | j        d           d S t          j                            | j                   d S d S t          | t          j                  rCt          | j                   | j        &t          j                            | j                   d S d S t          | t          j        t          j        t          j        f          rJt          j                            | j                   t          j                            | j                   d S t          | t          j        t          j        t          j        f          rWdt)          j        | j                  z  }|                                 D ]$}t          j                            || |           %d S t3          | d          r|                                  d S d S )Nheadmlpư>)stdg      ?init_weights)
isinstancennLinear
startswithinitzeros_weight	constant_biasr
   xavier_uniform_normal_Conv2d	LayerNormBatchNorm2d	GroupNormones_RNNGRULSTMmathsqrthidden_size
parametersuniform_hasattrr   )r   r   r   flaxstdvr%   s         Q/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/sequencer.py_init_weightsr;      sd   &")$$ ??6"" 	4GNN6=)))Gfk955555 4fm,,,;*GNN6;///// +* ''666;*}}>>>>>v{33333	 +*
 
FBI	&	& fm$$$;"GNN6;''''' #"	FR\2>2<H	I	I 
fm$$$
v{#####	FRVRVRW5	6	6 TYv1222'')) 	2 	2FGVdUD1111	2 	2		(	(      c                   R     e Zd Z fdZdej        deej        df         fdZ xZS )RNNIdentityc                 V    t          t          |                                            d S N)superr>   __init__)selfargskwargs	__class__s      r:   rB   zRNNIdentity.__init__<   s%    k4  ))+++++r<   xreturnNc                 
    |d fS r@    rC   rG   s     r:   forwardzRNNIdentity.forward?   s    $wr<   )	__name__
__module____qualname__rB   torchTensorr   rL   __classcell__rF   s   @r:   r>   r>   ;   sh        , , , , , %d0B*C        r<   r>   c                   F     e Zd Z	 	 	 	 	 ddededededef
 fd	Zd
 Z xZS )	RNN2dBaser   Tcat
input_sizer4   
num_layersr'   bidirectionalc                 J   t                                                       || _        || _        |rd|z  n|| _        || _        d| _        d| _        || _        d | _	        |r|dk    r$t          j        d| j        z  |          | _	        np|dk    r!t          j        | j        |          | _	        nI|dk    r(t          j        | j        |          | _	        d| _        n|dk    r't          j        | j        |          | _	        d| _        nt          d|z             |dk    r.	 d| j        z  |k    rt          d	d| j        z   d
| d          n|dk    r(	 | j        |k    rt          d	| j         d
| d          nz|dk    r.| j        |k    rt          d	| j         d
| d          d| _        nF|dk    r.| j        |k    rt          d	| j         d
| d          d| _        nt          d|z             t                      | _        t                      | _        d S )N   TrV   addverticalF
horizontalzUnrecognized union: zThe output channel z% is different from the input channel .)rA   rB   rW   r4   output_sizeunionwith_verticalwith_horizontalwith_fcfcr    r!   
ValueErrorr>   rnn_vrnn_h	rC   rW   r4   rX   r'   rY   ra   rd   rF   s	           r:   rB   zRNN2dBase.__init__E   s~    	$&.;L1{??
!# 	=~~)A(8$8*EE%)D$4jAA*$$)D$4jAA',$$,&&)D$4jAA%*"" !7%!?@@@e^^4##z11 !q4;K7K!!r|!!!  A  A  A 2e^^:-- !{t7G!{!{nx!{!{!{||| .j  :-- !{t7G!{!{nx!{!{!{|||#(D  l"":-- !{t7G!{!{nx!{!{!{|||!&D3e;<<< ]]
 ]]


r<   c                 h   |j         \  }}}}| j        rx|                    dddd          }|                    d||          }|                     |          \  }}|                    |||d          }|                    dddd          }nd }| j        rH|                    d||          }|                     |          \  }}|                    |||d          }nd }|,|*| j        dk    rt          j	        ||gd          }n||z   }n	||}n||}| j
        | 
                    |          }|S )Nr   r[   r      rV   )dim)shaperb   permutereshaperg   rc   rh   ra   rP   rV   re   )	rC   rG   BHWCv_hs	            r:   rL   zRNN2dBase.forward~   sN   W
1a 			!Q1%%A		"a##A::a==DAq		!Q2&&A		!Q1%%AAA 			"a##A::a==DAq		!Q2&&AAA=Q]zU""Iq!f"---E]AA]A7

Ar<   r   TTrV   T)rM   rN   rO   intboolrB   rL   rR   rS   s   @r:   rU   rU   C   s          "&7# 7#7# 7# 	7#
 7#  7# 7# 7# 7# 7# 7#r             r<   rU   c                   @     e Zd Z	 	 	 	 	 d
dededededef
 fd	Z xZS )LSTM2dr   TrV   rW   r4   rX   r'   rY   c           	          t                                          |||||||           | j        rt          j        |||d||          | _        | j        r!t          j        |||d||          | _        d S d S )NT)batch_firstr'   rY   )rA   rB   rb   r    r1   rg   rc   rh   ri   s	           r:   rB   zLSTM2d.__init__   s     	[*dMSXZabbb 	 +  DJ  	 +  DJJJ	 	r<   rx   )rM   rN   rO   ry   rz   rB   rR   rS   s   @r:   r|   r|      s          "&   	
            r<   r|   c                   f     e Zd Zdee eej        d          ej        ddddddf fd	Z	d	 Z
 xZS )
Sequencer2dBlock      @r   epsr   TrV   r   c                 Z   t                                                       t          ||z            } ||          | _         |||||	|
|          | _        |dk    rt          |          nt          j                    | _         ||          | _	         |||||          | _
        d S )N)rX   rY   ra   rd   r   )	act_layerdrop)rA   rB   ry   norm1
rnn_tokensr   r    Identity	drop_pathnorm2mlp_channels)rC   rm   r4   	mlp_ratio	rnn_layer	mlp_layer
norm_layerr   rX   rY   ra   rd   r   r   channels_dimrF   s                  r:   rB   zSequencer2dBlock.__init__   s      	9s?++Z__
#)!'
 
 
 1:B),,,BKMMZ__
%Ic<9SWXXXr<   c                     ||                      |                     |                     |                              z   }||                      |                     |                     |                              z   }|S r@   )r   r   r   r   r   rK   s     r:   rL   zSequencer2dBlock.forward   sa    ttzz!}}==>>>t00A??@@@r<   )rM   rN   rO   r|   r   r   r    r+   GELUrB   rL   rR   rS   s   @r:   r   r      s        
 wr|666gY Y Y Y Y Y>      r<   r   c                   $     e Zd Z fdZd Z xZS )Shufflec                 H    t                                                       d S r@   )rA   rB   )rC   rF   s    r:   rB   zShuffle.__init__   s    r<   c                     | j         r_|j        \  }}}}t          j        ||z            }|                    |d|          }|d d |d d f                             |||d          }|S )Nrl   )trainingrn   rP   randpermrp   )rC   rG   rq   rr   rs   rt   rs          r:   rL   zShuffle.forward   ss    = 	0JAq!Qq1u%%A		!R##A!!!Q'
""1aB//Ar<   rM   rN   rO   rB   rL   rR   rS   s   @r:   r   r      sG                  r<   r   c                   $     e Zd Z fdZd Z xZS )Downsample2dc                     t                                                       t          j        ||||          | _        d S )N)kernel_sizestride)rA   rB   r    r*   down)rC   	input_dim
output_dim
patch_sizerF   s       r:   rB   zDownsample2d.__init__   s8    IiT^___			r<   c                     |                     dddd          }|                     |          }|                     dddd          }|S )Nr   rk   r   r[   )ro   r   rK   s     r:   rL   zDownsample2d.forward   sD    IIaAq!!IIaLLIIaAq!!r<   r   rS   s   @r:   r   r      sL        ` ` ` ` `      r<   r   c                   h     e Zd Zdeee eej        d          ej	        ddddddf fd	Z
d	 Z xZS )
Sequencer2dStageFr   r   r   TrV   r   c                    t                                                       |rt          |||          | _        n ||k    sJ t	          j                    | _        g }t          |          D ]Q}|                     |||||	|
|||||||t          |t          t          f          r||         n|                     Rt	          j        | | _        d S )N)r   r   r   r   r   rX   rY   ra   rd   r   r   )rA   rB   r   
downsampler    r   rangeappendr   listtuple
Sequentialblocks)rC   rm   dim_outdepthr   r4   r   r   block_layerr   r   r   r   rX   rY   ra   rd   r   r   r   	block_idxrF   s                        r:   rB   zSequencer2dStage.__init__  s    * 	 	,*3DDDOO'>>>> kmmDOu 	 	IMM++###%#%+2<Yu2V2Ve)I..\e       mV,r<   c                 Z    |                      |          }|                     |          }|S r@   )r   r   rK   s     r:   rL   zSequencer2dStage.forward1  s'    OOAKKNNr<   )rM   rN   rO   r   r|   r   r   r    r+   r   rB   rL   rR   rS   s   @r:   r   r     s         (wr|666g'-- -- -- -- -- --^      r<   r   c                   H    e Zd Zddddddddd	eee eej        d
          ej	        ddddddddf fd	Z
ddZej        j        dd            Zej        j        d d            Zej        j        dej        fd            Zd!dedee         fdZd ZddefdZd Z xZS )"r        rk   avg   rk      rk   )   r[   r[   r        r   r   0   `   r   r   r   r   r   r   r   r   r   TrV   r   Fc           	      >   t                                                       |dv sJ || _        || _        |d         x| _        | _        d| _        d| _        g | _        t          d |d         ||d         |r|nd dd          | _
        t          |          t          |          cxk    r;t          |          cxk    r't          |          cxk    rt          |	          k    sn J t          t          |d                     }g }|d         }t          |          D ]\  }}|t          |||         fi d||         d	|dk    d
||         d||         d|	|         d|
d|d|d|d|d|d|d|d|d|d|gz  }||         }| xj        t!          |||         d|           gz  c_        t#          j        | | _         ||d                   | _        t+          | j        |||| j                  | _        |                     |           d S )N) r   rl   NHWCr   F)img_sizer   in_chans	embed_dimr   flatten
output_fmtc                     | |z  S r@   rJ   )rG   ys     r:   <lambda>z&Sequencer2d.__init__.<locals>.<lambda>e  s
    q1u r<   r   r   r   r4   r   r   r   r   r   r   rX   rY   ra   rd   r   r   stages.)num_chs	reductionr   )	pool_type	drop_rate	input_fmt)nlhb)rA   rB   num_classesglobal_poolnum_featureshead_hidden_sizefeature_dimr   feature_infor   stemlenr   r   	enumerater   dictr    r   stagesnormr   r   r   )rC   r   r   r   r   layerspatch_sizes
embed_dimshidden_sizes
mlp_ratiosr   r   r   r   r   num_rnn_layersrY   ra   rd   r   drop_path_rater   	stem_norm
reductionsr   prev_dimirv   rF   s                               r:   rB   zSequencer2d.__init__8  s   2 	k))))&&4>rNBD1 "1~ m%.8zzD
 
 
	 6{{c+..iiii#j//iiiiSEVEViiiiZ]^hZiZiiiiiii*[2D2DEEFF
a=j)) 	i 	iDAq'1   Qii q55	
 'q>> )OO %Q-- (K $) $) &: $) *> ,m e   !" Y#$ ).%  F( "!}H$x:a=YfcdYfYf"g"g"g!hhmV,Jz"~..	"!o
 
 
	 	t$$$$$r<   c                     |rt          j        | j                   nd}t          t	          t
          |          |            d S )Nr   )r   )r   )r2   logr   r   r   r;   )rC   r   r   s      r:   r   zSequencer2d.init_weights  sG    37?TXd.////R	GMY???MMMMMMr<   c                 4    t          d|rddgng d          S )Nz^stem)z^stages\.(\d+)Nz^norm)i ))z^stages\.(\d+)\.blocks\.(\d+)N)z^stages\.(\d+)\.downsample)r   r   )r   r   )r   )rC   coarses     r:   group_matcherzSequencer2d.group_matcher  sE     )$   

 

 

 
	
r<   c                     |r
J d            d S )Nz$gradient checkpointing not supportedrJ   )rC   enables     r:   set_grad_checkpointingz"Sequencer2d.set_grad_checkpointing  s    AAAAAAAAr<   rH   c                     | j         S r@   r   )rC   s    r:   get_classifierzSequencer2d.get_classifier  s
    yr<   Nr   r   c                 L    || _         | j                            ||           d S )N)r   )r   r   reset)rC   r   r   s      r:   reset_classifierzSequencer2d.reset_classifier  s(    &	{;;;;;r<   c                     |                      |          }|                     |          }|                     |          }|S r@   )r   r   r   rK   s     r:   forward_featureszSequencer2d.forward_features  s4    IIaLLKKNNIIaLLr<   
pre_logitsc                 ^    |r|                      |d          n|                      |          S )NT)r   r   )rC   rG   r   s      r:   forward_headzSequencer2d.forward_head  s,    0:Ltyyty,,,		!Lr<   c                 Z    |                      |          }|                     |          }|S r@   )r   r   rK   s     r:   rL   zSequencer2d.forward  s-    !!!$$a  r<   F)Tr@   )rM   rN   rO   r   r|   r   r   r    r+   r   rB   r   rP   jitignorer   r   Moduler   ry   r   strr   r   rz   r   rL   rR   rS   s   @r:   r   r   7  s        $+)+(wr|666g/R% R% R% R% R% R%hN N N N Y
 
 
 
 YB B B B Y	    < <C <hsm < < < <  M M$ M M M M      r<   c                     d| v r| S d| v r| d         } ddl }i }|                                 D ]O\  }}|                    dd |          }|                    dd|          }|                    d	d
          }|||<   P|S )z$ Remap original checkpoints -> timm zstages.0.blocks.0.norm1.weightmodelr   Nzblocks.([0-9]+).([0-9]+).downc                 T    dt          |                     d                    dz    dS )Nr   r   z.downsample.down)ry   group)rG   s    r:   r   z&checkpoint_filter_fn.<locals>.<lambda>  s)    ?nQWWUVZZ[\I\?n?n?n r<   zblocks.([0-9]+).([0-9]+)zstages.\1.blocks.\2zhead.zhead.fc.)reitemssubreplace)
state_dictr  r  out_dictkru   s         r:   checkpoint_filter_fnr    s    ':55*(
IIIH  ""  1FF35n5npqrrFF.0FJJIIgz**Or<   c                     t          t          d                    }|                    d|          }t          t          | |ft
          t          d|          d|}|S )Nrk   out_indicesT)flatten_sequentialr  )pretrained_filter_fnfeature_cfg)r   r   popr   r   r  r   )variant
pretrainedrE   default_out_indicesr  r  s         r:   _create_sequencer2dr    sp    a//**],?@@K  2DkJJJ   E Lr<   r   c                 @    | ddd t           ddt          t          ddd|S )Nr   )rk   r   r   bicubicTz	stem.projzhead.fc)urlr   rW   	pool_sizecrop_pctinterpolationfixed_input_sizemeanr   
first_conv
classifier)r	   r   r   )r  rE   s     r:   _cfgr'    s8    =t$yVZ%.B!   r<   ztimm/)	hf_hub_id)zsequencer2d_s.in1kzsequencer2d_m.in1kzsequencer2d_l.in1krH   c                     t          g dg dg dg dg dt          ddd	  	        }t          dd
| it          |fi |}|S )Nr   r   r[   r   r   r   r   r   TrV   	r   r   r   r   r   r   rY   ra   rd   sequencer2d_sr  )r,  r   r|   r  r  rE   
model_argsr  s       r:   r,  r,    s{    || LL'''%%%'''
 
 
J  eeJe$zJdJd]cJdJdeeELr<   c                     t          dg dg dg dg dg dt          dddd	|}t          dd
| it          |fi |}|S )N)r   rk      rk   r*  r   r   r   TrV   r+  sequencer2d_mr  rJ   )r2  r-  r.  s       r:   r2  r2         
}} LL'''%%%'''
 
 
 
J  eeJe$zJdJd]cJdJdeeELr<   c                     t          dg dg dg dg dg dt          dddd	|}t          dd
| it          |fi |}|S )N)r   r      r   r*  r   r   r   TrV   r+  sequencer2d_lr  rJ   )r6  r-  r.  s       r:   r6  r6    r3  r<   )r   Fr  )r   )1__doc__r2   	functoolsr   	itertoolsr   typingr   r   rP   torch.nnr    	timm.datar   r   r	   timm.layersr
   r   r   r   r   _builderr   _manipulater   	_registryr   r   __all__r  r  floatr;   r>   rU   r|   r   r   r   r   r   r  r  r'  default_cfgsr,  r2  r6  rJ   r<   r:   <module>rD     s                       " " " " " " " "        S S S S S S S S S S P P P P P P P P P P P P P P * * * * * * $ $ $ $ $ $ < < < < < < < </ ") 3 5    B    ")   [ [ [ [ [	 [ [ [|    Y   B# # # # #ry # # #L
 
 
 
 
bi 
 
 
	 	 	 	 	29 	 	 	3 3 3 3 3ry 3 3 3l    ")   D  $       %$$111$111$111& &                        r<   