
    Ngj                        d Z ddlZddlmZ ddlmZmZmZmZ ddl	Z	ddl
mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZm Z m!Z! ddgZ" G d dej#                  Z$ G d dej#                  Z% G d dej#                  Z& G d dej#                  Z' G d dej#                  Z( G d dej#                  Z)dkdej#        de*de+fdZ,d Z-dldZ.dmd!Z/ ei d" e/            d# e/            d$ e/            d% e/d&d'(          d) e/d&d*d+,          d- e/            d. e/d&d/(          d0 e/d&d1d+,          d2 e/d&d3d4d5d6d7d89          d: e/d&d;d4d5d6d7<          d= e/ee>          d? e/d&d@eeA          dB e/d&dCeeA          dD e/d&dEeeA          dF e/d&dGeeA          dH e/d&dIeeA          dJ e/d&dKeeA           e/d&dLeeA           e/d&dMeeA           e/d&dNeeA           e/d&dOeeA           e/d&dPeeA           e/d&dQeeA           e/             e/d&dR(           e/            dS	          Z0e dldTe)fdU            Z1e dldTe)fdV            Z2e dldTe)fdW            Z3e dldTe)fdX            Z4e dldTe)fdY            Z5e dldTe)fdZ            Z6e dldTe)fd[            Z7e dldTe)fd\            Z8e dldTe)fd]            Z9e dldTe)fd^            Z:e dldTe)fd_            Z;e dldTe)fd`            Z<e dldTe)fda            Z=e dldTe)fdb            Z>e dldTe)fdc            Z? e!e@d%d.d:d2dJdddedfdgdhdidj           dS )nuj   MLP-Mixer, ResMLP, and gMLP in PyTorch

This impl originally based on MLP-Mixer paper.

Official JAX impl: https://github.com/google-research/vision_transformer/blob/linen/vit_jax/models_mixer.py

Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601

@article{tolstikhin2021,
  title={MLP-Mixer: An all-MLP Architecture for Vision},
  author={Tolstikhin, Ilya and Houlsby, Neil and Kolesnikov, Alexander and Beyer, Lucas and Zhai, Xiaohua and Unterthiner,
        Thomas and Yung, Jessica and Keysers, Daniel and Uszkoreit, Jakob and Lucic, Mario and Dosovitskiy, Alexey},
  journal={arXiv preprint arXiv:2105.01601},
  year={2021}
}

Also supporting ResMlp, and a preliminary (not verified) implementations of gMLP

Code: https://github.com/facebookresearch/deit
Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
@misc{touvron2021resmlp,
      title={ResMLP: Feedforward networks for image classification with data-efficient training},
      author={Hugo Touvron and Piotr Bojanowski and Mathilde Caron and Matthieu Cord and Alaaeldin El-Nouby and
        Edouard Grave and Armand Joulin and Gabriel Synnaeve and Jakob Verbeek and Hervé Jégou},
      year={2021},
      eprint={2105.03404},
}

Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
@misc{liu2021pay,
      title={Pay Attention to MLPs},
      author={Hanxiao Liu and Zihang Dai and David R. So and Quoc V. Le},
      year={2021},
      eprint={2105.08050},
}

A thank you to paper authors for releasing code and weights.

Hacked together by / Copyright 2021 Ross Wightman
    N)partial)ListOptionalUnionTuple)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpGluMlpGatedMlpDropPathlecun_normal_	to_2tuple   )build_model_with_cfg)feature_take_indices)named_applycheckpoint_seq)generate_default_cfgsregister_modelregister_model_deprecations
MixerBlockMlpMixerc                   `     e Zd ZdZde eej        d          ej        ddf fd	Z	d Z
 xZS )r   z Residual Block w/ token mixing and channel MLPs
    Based on: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
          ?      @ư>eps        c	                 p   t                                                       fdt          |          D             \  }	}
 |          | _         |||	||          | _        |dk    rt          |          nt          j                    | _         |          | _	         ||
||          | _
        d S )Nc                 4    g | ]}t          |z            S  )int).0xdims     Q/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/mlp_mixer.py
<listcomp>z'MixerBlock.__init__.<locals>.<listcomp>J   s#    #O#O#OQCCLL#O#O#O    	act_layerdropr"   )super__init__r   norm1
mlp_tokensr   nnIdentity	drop_pathnorm2mlp_channels)selfr)   seq_len	mlp_ratio	mlp_layer
norm_layerr.   r/   r6   
tokens_dimchannels_dim	__class__s    `         r*   r1   zMixerBlock.__init__>   s     	#O#O#O#O)I:N:N#O#O#O 
LZ__
#)GZ9SWXXX09B),,,BKMMZ__
%Ic<9SWXXXr,   c           	      N   ||                      |                     |                     |                              dd                                        dd                    z   }||                      |                     |                     |                              z   }|S Nr      )r6   r3   r2   	transposer8   r7   r9   r(   s     r*   forwardzMixerBlock.forwardQ   s    ttzz!}}/F/Fq!/L/LMMWWXY[\]]^^^t00A??@@@r,   )__name__
__module____qualname____doc__r   r   r4   	LayerNormGELUr1   rF   __classcell__r@   s   @r*   r   r   :   s          !wr|666gY Y Y Y Y Y&      r,   c                   $     e Zd Z fdZd Z xZS )Affinec                     t                                                       t          j        t	          j        dd|f                    | _        t          j        t	          j        dd|f                    | _        d S )Nr   )	r0   r1   r4   	Parametertorchonesalphazerosbeta)r9   r)   r@   s     r*   r1   zAffine.__init__X   s_    \%*aC["9"9::
LaC[!9!9::			r,   c                 B    t          j        | j        | j        |          S N)rS   addcmulrW   rU   rE   s     r*   rF   zAffine.forward]   s    }TY
A666r,   )rG   rH   rI   r1   rF   rM   rN   s   @r*   rP   rP   W   sG        ; ; ; ; ;
7 7 7 7 7 7 7r,   rP   c                   B     e Zd ZdZdeeej        dddf fd	Zd Z	 xZ
S )ResBlockz Residual MLP block w/ LayerScale and Affine 'norm'

    Based on: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
       g-C6?r"   c
                    t                                                       t          ||z            }
 ||          | _        t	          j        ||          | _        |	dk    rt          |	          nt	          j                    | _	         ||          | _
         |||
||          | _        t	          j        |t          j        |          z            | _        t	          j        |t          j        |          z            | _        d S )Nr"   r-   )r0   r1   r&   r2   r4   Linearlinear_tokensr   r5   r6   r7   r8   rR   rS   rT   ls1ls2)r9   r)   r:   r;   r<   r=   r.   init_valuesr/   r6   channel_dimr@   s              r*   r1   zResBlock.__init__f   s     	#	/**Z__
Yw8809B),,,BKMMZ__
%Ic;)RVWWW<ejoo =>><ejoo =>>r,   c           
      n   ||                      | j        |                     |                     |                              dd                                        dd          z            z   }||                      | j        |                     |                     |                    z            z   }|S rB   )r6   ra   r`   r2   rD   rb   r8   r7   rE   s     r*   rF   zResBlock.forward|   s    tx$*<*<TZZ]]=T=TUVXY=Z=Z*[*[*e*efgij*k*kkllltx$*;*;DJJqMM*J*JJKKKr,   )rG   rH   rI   rJ   r   rP   r4   rL   r1   rF   rM   rN   s   @r*   r\   r\   a   so          g? ? ? ? ? ?,      r,   r\   c                   <     e Zd ZdZej        f fd	Zd Zd Z xZ	S )SpatialGatingUnitzc Spatial Gating Unit

    Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    c                     t                                                       |dz  } ||          | _        t          j        ||          | _        d S )NrC   )r0   r1   normr4   r_   proj)r9   r)   r:   r=   gate_dimr@   s        r*   r1   zSpatialGatingUnit.__init__   sJ    !8Jx((	Igw//			r,   c                     t           j                            | j        j        d           t           j                            | j        j                   d S )Nr   std)r4   initnormal_rj   weightones_biasr9   s    r*   init_weightszSpatialGatingUnit.init_weights   s=    
	(d333
din%%%%%r,   c                     |                     dd          \  }}|                     |          }|                     |                    dd                    }||                    dd          z  S )NrC   r)   )chunkri   rj   rD   )r9   r(   uvs       r*   rF   zSpatialGatingUnit.forward   sb    wwqbw!!1IIaLLIIakk"b))**1;;r2&&&&r,   )
rG   rH   rI   rJ   r4   rK   r1   ru   rF   rM   rN   s   @r*   rg   rg      sl          13 0 0 0 0 0 0& & &
' ' ' ' ' ' 'r,   rg   c                   `     e Zd ZdZde eej        d          ej        ddf fd	Z	d Z
 xZS )SpatialGatingBlockzp Residual Block w/ Spatial Gating

    Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    r]   r   r    r"   c	                 <   t                                                       t          ||z            }	 ||          | _        t	          t
          |          }
 |||	||
|          | _        |dk    rt          |          nt          j	                    | _
        d S )N)r:   )r.   
gate_layerr/   r"   )r0   r1   r&   ri   r   rg   r8   r   r4   r5   r6   )r9   r)   r:   r;   r<   r=   r.   r/   r6   rd   sgur@   s              r*   r1   zSpatialGatingBlock.__init__   s     	#	/**JsOO	'999%Ic;)X[bfggg09B),,,BKMMr,   c                     ||                      |                     |                     |                              z   }|S rY   )r6   r8   ri   rE   s     r*   rF   zSpatialGatingBlock.forward   s5    t001>>???r,   )rG   rH   rI   rJ   r   r   r4   rK   rL   r1   rF   rM   rN   s   @r*   r~   r~      s          wr|666gR R R R R R$      r,   r~   c                   R    e Zd Zdddddddee eej        d	          ej        d
d
d
dddf fd	Z	e
j        j        d(d            Ze
j        j        d(d            Ze
j        j        d)d            Ze
j        j        dej        fd            Zd*dedee         fdZ	 	 	 	 	 d+de
j        deeeee         f                  dededededeee
j                 ee
j        ee
j                 f         f         fdZ	 	 	 d,deeee         f         d!ed"efd#Zd$ Zd(d%efd&Zd' Z xZ S )-r                    r   r   r    r"   Favgc                    	
 t                                                       | _        | _        x _        x _         _        d _        t          ||||r
nd            _	        t           j	        d          r j	                                        n|t          j        	
 f	dt          |          D               _        fdt          |          D              _         
           _        t          j        |           _        |dk    rt          j         j                  nt          j                     _                             |           d S )NF)img_size
patch_sizein_chans	embed_dimr=   
feat_ratioc                 J   	 g | ]} 
j         j        	            S ))r<   r=   r.   r/   r6   )stemnum_patches)r'   _r.   block_layerdrop_path_rater   r<   r;   r=   proj_drop_rater9   s     r*   r+   z%MlpMixer.__init__.<locals>.<listcomp>   sY     &( &( &(  K	%#%##(	 	 	&( &( &(r,   c                 :    g | ]}t          d |           S )zblocks.)modulenum_chs	reductiondict)r'   ir   r   s     r*   r+   z%MlpMixer.__init__.<locals>.<listcomp>   sA     k k kSTD!yINNNk k kr,   r   )nlhb)r0   r1   num_classesglobal_poolnum_featureshead_hidden_sizer   grad_checkpointingr
   r   hasattrr   r4   
Sequentialrangeblocksfeature_infori   Dropout	head_dropr_   r5   headru   )r9   r   r   r   r   
num_blocksr   r;   r   r<   r=   r.   	drop_rater   r   r   	stem_normr   r   r@   s   `     `````` ``   @r*   r1   zMlpMixer.__init__   s   ( 	&&ENNND1DN"'!%.8zzD
 
 
	 /6di.N.N^DI((***T^	m &( &( &( &( &( &( &( &( &( &( &( &( :&&&( &( &( )k k k k kX]^hXiXik k kJy))	I..>IAooBIi)9:::SUS^S`S`	t$$$$$r,   c                     |rt          j        | j                   nd}t          t	          t
          |          |            d S )Nr"   )	head_bias)r   )mathlogr   r   r   _init_weights)r9   r   r   s      r*   ru   zMlpMixer.init_weights   sG    37?TXd.////R	GMY???MMMMMMr,   c                 (    t          dddg          S )Nz^stem)z^blocks\.(\d+)N)z^norm)i )r   r   r   )r9   coarses     r*   group_matcherzMlpMixer.group_matcher   s%    -/CD
 
 
 	
r,   Tc                     || _         d S rY   )r   )r9   enables     r*   set_grad_checkpointingzMlpMixer.set_grad_checkpointing   s    "(r,   returnc                     | j         S rY   )r   rt   s    r*   get_classifierzMlpMixer.get_classifier   s
    yr,   Nr   r   c                     || _         ||dv sJ || _        |dk    rt          j        | j        |          nt          j                    | _        d S )N) r   r   )r   r   r4   r_   r   r5   r   )r9   r   r   s      r*   reset_classifierzMlpMixer.reset_classifier  sY    &"+----*D>IAooBIdnk:::SUS^S`S`			r,   NCHWr(   indicesri   
stop_early
output_fmtintermediates_onlyc                 p   |dv s
J d            |dk    }g }t          t          | j                  |          \  }	}
|j        \  }}}|                     |          }t
          j                                        s|s| j        }n| j        d|
dz            }t          |          D ]@\  }} ||          }||	v r,|	                    |r| 
                    |          n|           A|r/| j                            ||f          \  fd|D             }|r|S | 
                    |          }||fS )a=   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r   NLCz)Output format must be one of NCHW or NLC.r   Nr   c                     g | ]B}|                     d                               dddd                                          CS )rw   r   r   r   rC   )reshapepermute
contiguous)r'   yBHWs     r*   r+   z2MlpMixer.forward_intermediates.<locals>.<listcomp>6  sL    lllYZQYYq!Q33;;Aq!QGGRRTTlllr,   )r   lenr   shaper   rS   jitis_scripting	enumerateappendri   dynamic_feat_size)r9   r(   r   ri   r   r   r   r   intermediatestake_indices	max_indexr   heightwidthr   r   blkr   r   r   s                    @@@r*   forward_intermediateszMlpMixer.forward_intermediates	  sv   , _,,,.Y,,,&"6s4;7G7G"Q"Qi  g1feIIaLL9!!## 	1: 	1[FF[)a-0F'' 	B 	BFAsAAL  $$T%@TYYq\\\qAAA  	m9..??DAqllllll^klllM 	!  IIaLL-r,   r   
prune_norm
prune_headc                     t          t          | j                  |          \  }}| j        d|dz            | _        |rt          j                    | _        |r|                     dd           |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )r   r   r   r4   r5   ri   r   )r9   r   r   r   r   r   s         r*   prune_intermediate_layersz"MlpMixer.prune_intermediate_layers?  sq     #7s4;7G7G"Q"Qik.9q=.1 	&DI 	)!!!R(((r,   c                     |                      |          }| j        r4t          j                                        st          | j        |          }n|                     |          }|                     |          }|S rY   )r   r   rS   r   r   r   r   ri   rE   s     r*   forward_featureszMlpMixer.forward_featuresO  sg    IIaLL" 	59+A+A+C+C 	t{A..AAAAIIaLLr,   
pre_logitsc                     | j         dk    r|                    d          }|                     |          }|r|n|                     |          S )Nr   r   rx   )r   meanr   r   )r9   r(   r   s      r*   forward_headzMlpMixer.forward_headX  sK    u$$1ANN10qqDIIaLL0r,   c                 Z    |                      |          }|                     |          }|S rY   )r   r   rE   s     r*   rF   zMlpMixer.forward^  s-    !!!$$a  r,   F)TrY   )NFFr   F)r   FT)!rG   rH   rI   r   r   r   r4   rK   rL   r1   rS   r   ignoreru   r   r   Moduler   r&   r   strr   Tensorr   r   boolr   r   r   r   r   rF   rM   rN   s   @r*   r   r      s         "wr|666g%5% 5% 5% 5% 5% 5%n YN N N N Y
 
 
 
 Y) ) ) ) Y	    a aC ahsm a a a a 8<$$',4  4 |4  eCcN344  	4 
 4  4  !%4  
tEL!5tEL7I)I#JJ	K4  4  4  4 p ./$#	 3S	>*  	      1 1$ 1 1 1 1      r,   r"   Fr   namer   c                    t          | t          j                  r$|                    d          rKt          j                            | j                   t          j                            | j        |           dS |rCt          | j                   | j        &t          j                            | j                   dS dS t          j        
                    | j                   | j        Rd|v r(t          j                            | j        d           dS t          j                            | j                   dS dS t          | t          j                  rCt          | j                   | j        &t          j                            | j                   dS dS t          | t          j        t          j        t          j        f          rJt          j                            | j                   t          j                            | j                   dS t#          | d          r|                                  dS dS )zA Mixer weight initialization (trying to match Flax defaults)
    r   Nmlpr   rm   ru   )
isinstancer4   r_   
startswithro   zeros_rq   	constant_rs   r   xavier_uniform_rp   Conv2drK   BatchNorm2d	GroupNormrr   r   ru   )r   r   r   flaxs       r*   r   r   d  s    &")$$ ??6"" 	4GNN6=)))Gfk955555 4fm,,,;*GNN6;///// +* ''666;*}}>>>>>v{33333	 +*
 
FBI	&	& 
fm$$$;"GNN6;''''' #"	FR\2>2<H	I	I 
fm$$$
v{#####		(	(  	 r,   c                    d| v ri }|                                  D ]\  }}|                    dd          }|                    dd          }|                    dd          }|                    dd	          }|                    d
          s|                    d          r|                    ddd          }|||<   |S | S )z Remap checkpoints if needed zpatch_embed.proj.weightzpatch_embed.zstem.zattn.zlinear_tokens.zmlp.zmlp_channels.gamma_lsz.alphaz.betar   rw   )itemsreplaceendswithr   )
state_dictmodelout_dictkr|   s        r*   checkpoint_filter_fnr    s     J..$$&& 	 	DAq		.'22A		'#344A		&/22A		(D))Azz(## (qzz'':': (IIaB''HQKKr,   c                     |                     dd          }t          t          | |ft          t	          |d          d|}|S )Nout_indicesr   getter)r  feature_cls)pretrained_filter_fnfeature_cfg)popr   r   r  r   )variant
pretrainedkwargsr  r  s        r*   _create_mixerr    s]    **]A..K  2[hGGG   E Lr,   r   c                 "    | ddd dddddddd	|S )
Nr   )r   r   r         ?bicubicT)r   r   r   z	stem.projr   )urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer   rn   
first_conv
classifierr%   )r  r  s     r*   _cfgr    s6    =tI4!   r,   zmixer_s32_224.untrainedzmixer_s16_224.untrainedzmixer_b32_224.untrainedz mixer_b16_224.goog_in21k_ft_in1kztimm/zlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224-76587d61.pth)	hf_hub_idr  zmixer_b16_224.goog_in21kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224_in21k-617b3de2.pthiSU  )r  r  r   zmixer_l32_224.untrainedz mixer_l16_224.goog_in21k_ft_in1kzlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224-92f9adc4.pthzmixer_l16_224.goog_in21kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224_in21k-846aa33c.pthzmixer_b16_224.miil_in21kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil_in21k-2a558a71.pth)r"   r"   r"   )      ?r  r  r  bilineari+  )r  r  r   rn   r  r  r   z mixer_b16_224.miil_in21k_ft_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil-9229a591.pth)r  r  r   rn   r  r  zgmixer_12_224.untrained)r   rn   zgmixer_24_224.ra3_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmixer_24_224_raa-7daf7ae6.pth)r  r  r   rn   zresmlp_12_224.fb_in1kz9https://dl.fbaipublicfiles.com/deit/resmlp_12_no_dist.pthzresmlp_24_224.fb_in1kz9https://dl.fbaipublicfiles.com/deit/resmlp_24_no_dist.pthzresmlp_36_224.fb_in1kz9https://dl.fbaipublicfiles.com/deit/resmlp_36_no_dist.pthzresmlp_big_24_224.fb_in1kz:https://dl.fbaipublicfiles.com/deit/resmlpB_24_no_dist.pthzresmlp_12_224.fb_distilled_in1kz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_36_dist.pthz7https://dl.fbaipublicfiles.com/deit/resmlpB_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlpB_24_22k.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dino.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dino.pthznhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmlp_s16_224_raa-10536d42.pth)	resmlp_24_224.fb_distilled_in1kresmlp_36_224.fb_distilled_in1k#resmlp_big_24_224.fb_distilled_in1k"resmlp_big_24_224.fb_in22k_ft_in1kzresmlp_12_224.fb_dinozresmlp_24_224.fb_dinozgmlp_ti16_224.untrainedzgmlp_s16_224.ra3_in1kzgmlp_b16_224.untrainedr   c                 B    t          ddddd|}t          dd| i|}|S )	zv Mixer-S/32 224x224
    Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
        r   r   r   r   r   mixer_s32_224r  r%   )r'  r   r  r  r  
model_argsr  s       r*   r'  r'    =    
 KSKKFKKJOOjOJOOELr,   c                 B    t          ddddd|}t          dd| i|}|S )	zw Mixer-S/16 224x224
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r   r   r   r&  mixer_s16_224r  r%   )r-  r(  r)  s       r*   r-  r-    r+  r,   c                 B    t          ddddd|}t          dd| i|}|S )	zw Mixer-B/32 224x224
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r%        r&  mixer_b32_224r  r%   )r1  r(  r)  s       r*   r1  r1  '  =    
 LcLLVLLJOOjOJOOELr,   c                 B    t          ddddd|}t          dd| i|}|S )	z Mixer-B/16 224x224. ImageNet-1k pretrained weights.
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r   r/  r0  r&  mixer_b16_224r  r%   )r4  r(  r)  s       r*   r4  r4  1  r2  r,   c                 B    t          ddddd|}t          dd| i|}|S )	zx Mixer-L/32 224x224.
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r%        r&  mixer_l32_224r  r%   )r8  r(  r)  s       r*   r8  r8  ;  =    
 MdMMfMMJOOjOJOOELr,   c                 B    t          ddddd|}t          dd| i|}|S )	z Mixer-L/16 224x224. ImageNet-1k pretrained weights.
    Paper:  'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601
    r   r6  r7  r&  mixer_l16_224r  r%   )r;  r(  r)  s       r*   r;  r;  E  r9  r,   c           
      f    t          dddddt          t          j        d|}t	          d	d| i|}|S )
zV Glu-Mixer-12 224x224
    Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer
    r   r/    r  r   r   r   r   r;   r<   r.   gmixer_12_224r  r%   )r@  r   r   r4   SiLUr  r)  s       r*   r@  r@  O  T    
  7"zBG7 7/57 7J OOjOJOOELr,   c           
      f    t          dddddt          t          j        d|}t	          d	d| i|}|S )
zV Glu-Mixer-24 224x224
    Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer
    r   r6  r=  r>  r?  gmixer_24_224r  r%   )rE  rA  r)  s       r*   rE  rE  [  rC  r,   c           
      \    t          dddddt          t          d|}t          d	d| i|}|S )
zx ResMLP-12
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r   r/  r=  r]   r   r   r   r;   r   r=   resmlp_12_224r  r%   )rH  )r   r\   rP   r  r)  s       r*   rH  rH  g  sU    
  u"qhciu umsu uJOOjOJOOELr,   c                 z    t          d
ddddt          t          d          t          d|}t	          dd	| i|}|S )zx ResMLP-24
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r   r6  r=  r]   gh㈵>rc   rG  resmlp_24_224r  r%   )rK  r   r   r\   rP   r  r)  s       r*   rK  rK  r  d    
  V"qH$777FV VNTV VJ OOjOJOOELr,   c                 z    t          d
ddddt          t          d          t          d|}t	          dd	| i|}|S )zx ResMLP-36
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r   $   r=  r]   r   rJ  rG  resmlp_36_224r  r%   )rP  rL  r)  s       r*   rP  rP  ~  rM  r,   c                 z    t          d
ddddt          t          d          t          d|}t	          dd	| i|}|S )zz ResMLP-B-24
    Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404
    r   r6  r0  r]   r   rJ  rG  resmlp_big_24_224r  r%   )rR  rL  r)  s       r*   rR  rR    sd    
  VsaH$777FV VNTV VJ SS*S
SSELr,   c           
      \    t          dddddt          t          d|}t          d	d| i|}|S )
zU gMLP-Tiny
    Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    r            r   r   r   r;   r   r<   gmlp_ti16_224r  r%   )rX  r   r~   r   r  r)  s       r*   rX  rX    sS    
  &"qN`& &$& &J OOjOJOOELr,   c           
      \    t          dddddt          t          d|}t          d	d| i|}|S )
zV gMLP-Small
    Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    r   rT     rV  rW  gmlp_s16_224r  r%   )r\  rY  r)  s       r*   r\  r\    S    
  &"qN`& &$& &J NNZN:NNELr,   c           
      \    t          dddddt          t          d|}t          d	d| i|}|S )
zU gMLP-Base
    Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050
    r   rT  r   rV  rW  gmlp_b16_224r  r%   )r_  rY  r)  s       r*   r_  r_    r]  r,   r   r!  r"  r#  rH  rK  )mixer_b16_224_in21kmixer_l16_224_in21kmixer_b16_224_miilmixer_b16_224_miil_in21kresmlp_12_distilled_224resmlp_24_distilled_224resmlp_36_distilled_224resmlp_big_24_distilled_224resmlp_big_24_224_in22ft1kresmlp_12_224_dinoresmlp_24_224_dino)r"   Fr   )r   )ArJ   r   	functoolsr   typingr   r   r   r   rS   torch.nnr4   	timm.datar   r	   timm.layersr
   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   r   __all__r   r   rP   r\   rg   r~   r   r   floatr   r  r  r  default_cfgsr'  r-  r1  r4  r8  r;  r@  rE  rH  rK  rP  rR  rX  r\  r_  rG   r%   r,   r*   <module>rw     s	  ' 'P        / / / / / / / / / / / /        A A A A A A A A ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] ] * * * * * * + + + + + + 4 4 4 4 4 4 4 4 Y Y Y Y Y Y Y Y Y Y
$       :7 7 7 7 7RY 7 7 7    ry   B' ' ' ' '	 ' ' '.       8l l l l lry l l l^ ") 3 5    D  "
 
 
 
    %$ a&ttvva&ttvva& ttvva& 'z) ) )	a&  A! ! !a& ttvva& 'z) ) )a&$  A! ! !%a&2  E|e:ch! ! !3a&< '~|e:) ) )=a&H tt)>DXYYYIa&J dd}"(<> > >Ka&T TTG"(<> > >Ua&\ TTG"(<	> > >]a&f TTG"(<> > >ga&n  H"(<"> "> ">oa&x &ttD"(<(> (> (>ya&@ (,tD"(<(> (> (> (,tD"(<(> (> (> ,04E"(<,> ,> ,>
 +/$D"(<+> +> +>
 "TD"(<> > > "TD"(<> > >
  $tvv!T|   #dffAa& a& a& a aH                                                                    X                        H==< :@@@#H"F))' '     r,   