
    Ng                    0   d Z ddlZddlmZmZmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZmZ ddlZddlmZ ddlmZmZmZmZ ddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( dd	l)m*Z* dd
l+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1m2Z2 g dZ3e G d d                      Z4e G d d                      Z5ddZ6ddZ7	 	 dde	e8e8f         dee9e
e9         f         de:de	e4         fdZ;dee4ee4         f         de
e4         fdZ<d  Z=e G d! d"                      Z> G d# d$ej?                  Z@d%e8d&e9d'e9d(e9d)e	e9e9f         d*e>fd+ZA G d, d-ej?                  ZB G d. d/ej?                  ZC G d0 d1ej?                  ZD G d2 d3ej?                  ZE G d4 d5ej?                  ZF G d6 d7ej?                  ZG G d8 d9ej?                  ZH eIeBeCeDeEeFeGeH:          ZJd;e8d<ej?        fd=ZKd>ee8ej?        f         fd?ZL G d@ dAejM                  ZN	 	 	 	 dd&e9d'e9dDe8dEe8dFe8d*e>fdGZOddIZPdJ ZQdKee8ef         dLe4dMe5fdNZRddeRfdOe5dPeSdQe9dRee8ef         dSee9         d*ee>         dTee         fdUZTddOe5dWe:fdXZU G dY dZej?                  ZVdd[ZW eIdi d\ e5 e4d]dd^dHdd_`           e4d]dHdadHdd_`           e4dbdcdddHdde`           e4dbdfdddHddg`           e4dbdhdddddg`          fdiddjk          dl e5 e4d]dd^dHdd_`           e4d]dHdadHdd_`           e4dbdcdddHdde`           e4dbdhdddHddg`           e4dbddddddg`          fdiddjk          dm e5 e4d]ddndHdd_`           e4d]dodndHdd_`           e4dbdpdqdHdde`           e4dbdHdrdHddg`           e4dbddsdddg`          fdtdduk          dv e5 e6dwdxy          dzdn{          d| e5 e6dwd}y          dzd~{          d e5 e6dwdy          dzd~{          d e5 e6d          dzd~{          d e5 e6d          dzd~{          d e5 e6ddh          dzd~{          d e5 e6d          dzd~{          d e5 e6ddh          dzd~{          d e5 e6d          dzd~{          d e5 e6ddh          dzd~{          d e5 e6ddy          dzd~d eIdd                    d e5 e4dbdHdsddide`           e4dbdhddHdide`           e4dbdcddHdide`           e4dbdhddHdd_`          fd^dddd          d e5 e4dddsddd_ eI                       e4dbdhddHdide`           e4dbdcddHdide`           e4dbdhddHdd_`          fd^dddd eIdV                    d e5 e4dbdHdsddide`           e4dbdHddHdide`           e4dbdHddHdide`           e4dbdHddHdide`          fd~ddd          d e5 e4dbdHdsddide`           e4dbdHddHdide`           e4dbdHddHdide`           e4dbdHddHdide`          fd~dddd          d e5 e4dbdHdsddide`           e4dbdHddHdide`           e4dbdHddHdide`           e4dbdHddHdide`          fd~dddd          d e5 e4dbdHdsddide`           e4dbdHddHdide`           e4dbdHddHdide`           e4dbdHddHdide`          fd~dddd          d e5 e4dbdHdsddide`           e4dbdHddHdide`           e4dbdHddHdide`           e4dbdHddHdide`          fd~dddd eId                    d e5 e4dbdHdsddde`           e4dbdoddHdde`           e4dbdoddHdde`           e4dbdHddHdde`          fd~ddBdd          d e5 e4dbdHdsddde`           e4dbdoddHdde`           e4dbdoddHdde`           e4dbdHddHdde`          fd~ddBdd          d e5 e4dbdHdsddde`           e4dbdoddHdde`           e4dbdoddHdde`           e4dbdHddHdde`          fd~ddBddd          d e5 e4dbdHdsddde`           e4dbdoddHdde`           e4dbdoddHdde`           e4dbdHddHdde`          fd~ddBddd          d e5 e4dbdHdsddde`           e4dbdoddHdde`           e4dbdoddHdde`           e4dbdHddHdde`          fd~ddBddd          d e5 e4dbdodsdde           e4dbdhddHde           e4dbdcddHde           e4dbdoddHde          fd~ddBd          d e5 e4dbdodsddide`           e4dbdhddHdide`           e4dbdcddHdide`           e4dbdoddHdide`          fd~dddd          d e5 e4dbdHdndHddo`           e4dbdcddHddo`           e4dbddadHddo`           e4dbdHddHddo`          fdidBdBddd eIde           eIdVdV¦          æ	  	        d e5 e4dbdHdndHddh`           e4dbdcddHddh`           e4dbddadHddh`           e4dbdHddHddh`          fdidBdBddd eIde           eIdVdV¦          æ	  	        d e5 e4dbdod~ddidh`           e4dbdcd^dHdidh`           e4dbddsdHdidh`           e4dbdodqdHdidh`          fd~ddBdBddd eIde           eIdVdV¦          Ǧ
  
        d e5 e4dbdod~dddh`           e4dbdcd^dHddh`           e4dbddsdHddh`           e4dbdodqdHddh`          fd~ddBdBddd eIde           eIdVdV¦          Ǧ
  
        d e5 e4dbdoddddh`           e4dbddadHddh`           e4dbddqdHddh`           e4dbdoddHddh`          fd~ddBdBddd eIde           eIdVdV¦          Ǧ
  
        d e5 e4dbdHdndHddo`           e4dbdcddHddo`           e4dbddadHddo`           e4dbdHddHddo`          fdidBdBdd eed˦          d eIde           eIdVdV¦          ̦
  
        d e5 e4dbdHdndHddh`           e4dbdcddHddh`           e4dbddadHddh`           e4dbdHddHddh`          fdidBdBdd eed˦          d eIde           eIdVdV¦          ̦
  
        d e5 e4dbdod~dddh`           e4dbdcd^dHddh`           e4dbddsdHddh`           e4dbdodqdHddh`          fd~ddBdBdd eed˦          d eIde           eIdVdV¦          Ц          d e5 e7ddhӦ          ddn{          d e5 e7d֬          dd~{          d e5 e7dج          dd~{          d e5 e7dڬ          dd~{          d e5 e7ddݬަ          dd~{          d e5 e4dbdodsdde           e4dbdhddHde           e4dbdcddHde           e4dbdoddHde          fddBdddd          d e5 e4dbdodsdde           e4dbdhddHde           e4dbdddHde           e4dbdoddHde          fddBdddd          d e5 e4dbdhdsdde           e4dbdcddHde           e4dbdddHde           e4dbdcddHde          fdddBdddd          d e5 e4dbdcdsdde           e4dbdddHde           e4dbdddHde           e4dbdddHde          fdddBdddd          d e5 e4dbdodsdde           e4dbdddHde           e4dbdddHde           e4dbdddHde          fdddBdddd          d e5 e4dbdodsdde           e4dbdhddHde           e4dbdcddHde           e4dbdoddHde          fddBddddd          d e5 e4dddidHdd`           e4ddd~dHdd`           e4d]dd^dHdide`           e4dbddsdHd~de`          fdddBdd eIde                    ZXdD ]ZY eeXeY         d          eXeYdz   <   	 dd ee8ejZ        f         deVde8dee8ejZ        f         fdZ[d ee8ejZ        f         deVfdZ\ddZ]ddZ^ddZ_ e1i d e^d	
          d e^d	
          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	dd          d e^d	ddddd_           d! e^d	d"d#ddd$d_%          d& e_d	d'd$d_(          d) e_d	d*d$d_(          i d+ e_d	d,d$d_(          d- e_d	d.d$d_(          d/ e_d	d0d$d_(          d1 e_d	d2d3          d4 e_d	d5d$d_(          d6 e_d	d7d$d_(          d8 e_d	d9d$d_(          d: e_d	d;d$d_(          d< e_d	d=d$d_(          d> e_d	d?d$d_(          d@ e_d	dAd$d_(          dB e_d	dCdDdEdEdFdGdHd$d_I
  
        dJ e_d	dKdDdEdEdHdd_L          dM e_d	dNdEdEdOdP          dQ e_d	dRdEdEdHdd_S          dT e_d	dUdEdEdHdd_S          dV e_dDdEdEdFdGdOd$W          i dX e_d	dYdDdEdEdOdZ          d[ e_d	d\dEdEdOdd_S          d] e^d	d^d_`          da e^d	dbd_`          dc e^d	dbd_`          dd e^d	dbd_`          de e^d	dbd_`          df e_d	dgdeedVdFdGdhi	  	        dj e_d	dgdeedVdFdGdhi	  	        dk e_d	dgddeedVd$dldhi	  	        dm e_d	dgdneedVdodpdhi	  	        dq e_d	dgdeedVdrdsdhi	  	        dt e_dudgdeedFdGv          dw e_dxdgdeedFdGv          dy e_dzdgdeed$dlv          d{ e_d|dgdeedodpv          d} e_d~dgdeedrdsv           e_dd           e_d	dDddOd          d          Z`e2ddeVfd            Zae2ddeVfd            Zbe2ddeVfd            Zce2ddeVfd            Zde2ddeVfd            Zee2ddeVfd            Zfe2ddeVfd            Zge2ddeVfd            Zhe2ddeVfd            Zie2ddeVfd            Zje2ddeVfd            Zke2ddeVfd            Zle2ddeVfd            Zme2ddeVfd            Zne2ddeVfd            Zoe2ddeVfd            Zpe2ddeVfd            Zqe2ddeVfd            Zre2ddeVfd            Zse2ddeVfd            Zte2ddeVfd            Zue2ddeVfd            Zve2ddeVfd            Zwe2ddeVfd            Zxe2ddeVfd            Zye2ddeVfd            Zze2ddeVfd            Z{e2ddeVfd            Z|e2ddeVfd            Z}e2ddeVfd            Z~e2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            Ze2ddeVfd            ZdS (  a   Bring-Your-Own-Blocks Network

A flexible network w/ dataclass based config for stacking those NN blocks.

This model is currently used to implement the following networks:

GPU Efficient (ResNets) - gernet_l/m/s (original versions called genet, but this was already used (by SENet author)).
Paper: `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
Code and weights: https://github.com/idstcv/GPU-Efficient-Networks, licensed Apache 2.0

RepVGG - repvgg_*
Paper: `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
Code and weights: https://github.com/DingXiaoH/RepVGG, licensed MIT

MobileOne - mobileone_*
Paper: `MobileOne: An Improved One millisecond Mobile Backbone` - https://arxiv.org/abs/2206.04040
Code and weights: https://github.com/apple/ml-mobileone, licensed MIT

In all cases the models have been modified to fit within the design of ByobNet. I've remapped
the original weights and verified accuracies.

For GPU Efficient nets, I used the original names for the blocks since they were for the most part
the same as original residual blocks in ResNe(X)t, DarkNet, and other existing models. Note also some
changes introduced in RegNet were also present in the stem and bottleneck blocks for this model.

A significant number of different network archs can be implemented here, including variants of the
above nets that include attention.

Hacked together by / copyright Ross Wightman, 2021.
    N)	dataclassfieldreplace)partial)TupleListDictOptionalUnionAnyCallableSequence)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDOPENAI_CLIP_MEANOPENAI_CLIP_STD)ClassifierHeadNormMlpClassifierHeadConvNormActBatchNormAct2dEvoNorm2dS0aAttentionPool2dRotAttentionPool2dDropPathAvgPool2dSamecreate_conv2dget_act_layerget_norm_act_layerget_attnmake_divisible	to_2tuple   )build_model_with_cfg)feature_take_indices)named_applycheckpoint_seq)generate_default_cfgsregister_model)ByobNetByoModelCfgByoBlockCfgcreate_byob_stemcreate_blockc                   @   e Zd ZU eeej        f         ed<   eed<   eed<   dZ	eed<   dZ
eeeef                  ed<   dZeed	<   dZee         ed
<   dZeeeef                  ed<   dZee         ed<   dZeeeef                  ed<   dZeeeef                  ed<   dS )r+   typedc   sNgs      ?br
attn_layerattn_kwargsself_attn_layerself_attn_kwargsblock_kwargs)__name__
__module____qualname__r   strnnModule__annotations__intr3   r4   r
   r   r6   floatr7   r8   r	   r   r9   r:   r;        O/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/byobnet.pyr+   r+   5   s         
RY

FFF
FFFAsJJJ)-BsH}%&---BNNN !%J$$$,0K$sCx.)000%)OXc])))15htCH~.555-1L(4S>*11111rF   r+   c                      e Zd ZU eeeeedf         f         df         ed<   dZeed<   dZ	eed<   dZ
ee         ed<   d	Zeeee         eedf         f         ed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee         ed<   dZeed<   dZee         ed<    ed           Zeed <   dZee         ed!<    ed"           Zeed#<    ed$           Z e!ee"f         ed%<   dS )&r*   .blocksconv1x1
downsample3x3	stem_typemaxpool	stem_pool    stem_chsr5   width_factorr   num_featuresTzero_init_lastFfixed_input_sizerelu	act_layer	batchnorm
norm_layer aa_layerNhead_hidden_size
classifier	head_typer7   c                      t                      S NdictrE   rF   rG   <lambda>zByoModelCfg.<lambda>^   s
    dff rF   )default_factoryr8   r9   c                      t                      S r`   ra   rE   rF   rG   rc   zByoModelCfg.<lambda>`   s
    466 rF   r:   c                      t                      S r`   ra   rE   rF   rG   rc   zByoModelCfg.<lambda>a   s
     rF   r;   )#r<   r=   r>   r   r   r+   rB   rK   r?   rM   rO   r
   rQ   rC   r   rR   rD   rS   rT   boolrU   rW   rY   r[   r\   r^   r7   r   r8   rb   r9   r:   r;   r	   r   rE   rF   rG   r*   r*   F   s        %U;+;%<<=sBCCCCJIs(Ix}(((79HeCcE#s(O34999L%L#ND"d""" Is!J!!!Hc '+hsm***!Is!!! !%J$$$nn===K===%)OXc])))"U>>BBBdBBB#(5#H#H#HL$sCx.HHHHHrF   r*            r"   r5   r5   r5   r5   c                 |    d}ddk    rfdt          fdt          | ||          D                       }|S )N@            r   c                 *    |dz   dz  dk    r| z  ndS )Nr"   r2   r   rE   )chsidxgroupss     rG   rc   z_rep_vgg_bcfg.<locals>.<lambda>h   s!    a1}7I7IcVmmq rF   c                 D    g | ]\  }}}t          d |||z            S )rep)r/   r0   r1   r4   )r+   ).0r0   r1   wf
group_sizes       rG   
<listcomp>z!_rep_vgg_bcfg.<locals>.<listcomp>i   s5    fffHAqRT+5ARJGGGfffrF   )tuplezip)r0   rz   rv   r1   bcfgr{   s     `  @rG   _rep_vgg_bcfgr   d   s\    AJzzPPPP
ffffX[\]_`bdXeXefffggDKrF   r2      
   r"   rE   c                    d}t          d|d         |d         z            }|pdt          |           z  }g }t          | |||          D ]\  } }}}g }	t          |           D ]a}
||z  }t	          |          }i }|
| |z
  k    rd|d<   |	t          ddd	|d	|d
|gz  }	|	t          ddd	|dt	          ddd	i|d
|gz  }	|}b||	gz  }|S )Nrn   ro   r   )r   )num_conv_branchesser7   oner"   )r/   r0   r1   r4   r;   kernel_sizerE   )minlenr~   rangerb   r+   )r0   rz   	se_blocksr   r1   prev_cr   wr   scfgiout_cbkaks                 rG   _mobileone_bcfgr   m   sE   AQqTBqE\""F*TCFF]ID1aY//  1aq 		 		AEE(9:::BBAF{{#'< [XeqFqrXXUWXXYYD[ ^a5QT=V=Va=VSU=V=V^ ^Z\^ ^ _ _DFFKrF   Ftypeseveryfirstreturnc                 b   t          |           dk    sJ t          |t                    r-t          t	          |rdn|||dz                       }|s|dz
  g}t          |           g }t	          |          D ])}||v r| d         n| d         }|t          d|dd|gz  }*t          |          S )z' interleave 2 block types in stack
    r2   r   r"   )r/   r0   rE   )r   
isinstancerC   listr   setr+   r}   )r   r0   r   r   kwargsrI   r   
block_types           rG   interleave_blocksr      s     u::????% U0115!UQY??@@ 	UGEJJJF1XX @ @!"eU1XXq
;>J!>>v>>??==rF   stage_blocks_cfgc                     t          | t                    s| f} g }t          |           D ](\  }|fdt          j                  D             z  })|S )Nc                 2    g | ]}t          d           S )r"   r0   )r   )ry   _cfgs     rG   r|   z%expand_blocks_cfg.<locals>.<listcomp>   s&    ???Qwsa(((???rF   )r   r   	enumerater   r0   )r   
block_cfgsr   r   s      @rG   expand_blocks_cfgr      sq    &11 /,.J,-- @ @3????%,,????

rF   c                 *    | sdS || z  dk    sJ || z  S )Nr"   r   rE   )r{   channelss     rG   
num_groupsr      s2     &q *$)))):%%rF   c                   x    e Zd ZU eZeed<   eZeed<   e	j
        Zeed<   dZee         ed<   dZee         ed<   dS )LayerFnconv_norm_actnorm_actactNattn	self_attn)r<   r=   r>   r   r   r   rB   r   r   r@   ReLUr   r   r
   r   rE   rF   rG   r   r      sn         )M8)))'Hh'''GC#D(8
###$(Ix!(((((rF   r   c                   H     e Zd Z	 	 	 	 ddededededed	ef fd
Zd Z xZS )DownsampleAvgr"   FNin_chsout_chsstridedilation	apply_actlayersc                 v   t          t          |                                            |pt                      }|dk    r|nd}|dk    s|dk    r4|dk    r|dk    rt          nt
          j        } |d|dd          | _        nt          j                    | _        |	                    ||d|          | _
        dS )z0 AvgPool Downsampling as in 'D' ResNet variants.r"   r2   TF)	ceil_modecount_include_padr   N)superr   __init__r   r   r@   	AvgPool2dpoolIdentityr   conv)
selfr   r   r   r   r   r   
avg_strideavg_pool_fn	__class__s
            rG   r   zDownsampleAvg.__init__   s     	mT""++---$799'1}}VV!
A::A+5??x!||--QSQ]K#AzTUZ[[[DIIDI((!y(QQ			rF   c                 R    |                      |                     |                    S r`   )r   r   r   xs     rG   forwardzDownsampleAvg.forward   s    yy1&&&rF   )r"   r"   FN)	r<   r=   r>   rC   rg   r   r   r   __classcell__r   s   @rG   r   r      s        
 #"R RR R 	R
 R R R R R R R R(' ' ' ' ' ' 'rF   r   downsample_typer   r   r   r   r   c                     | dv sJ ||k    s|dk    s|d         |d         k    r:| sd S | dk    rt          ||f||d         d|S  |j        ||fd||d         d|S t          j                    S )N)avgrJ   rZ   r"   r   r   r   r   )r   r   r   )r   r   r@   r   )r   r   r   r   r   r   r   s          rG   create_shortcutr      s     44444FaKK8A;(1++E+E 	w4%% `(ST+``Y_```'6'vQv`hij`kvvouvvv{}}rF   c                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 ddededededeeef         dee         dedede	de	de
dedef fdZdde	fdZd Z xZS )
BasicBlockz$ ResNet Basic Block - kxk + kxk
       r"   r"   r"   Nr5   r   TF        r   r   r   r   r   r{   bottle_ratiorK   	attn_last
linear_outr   
drop_blockdrop_path_ratec           	         t          t          |                                            |pt                      }t	          ||z            }t          ||          }t          |||||d|          | _        |                    |||||d                   | _	        |	s|j
        t          j                    n|
                    |          | _
        |                    ||||d         ||d          | _        |	r|j
        t          j                    n|
                    |          | _        |dk    rt          |          nt          j                    | _        |
rt          j                    n|                    d	          | _        d S )
NFr   r   r   r   r   r   r"   )r   rv   
drop_layerr   r   Tinplace)r   r   r   r   r    r   r   shortcutr   	conv1_kxkr   r@   r   	conv2_kxkr   r   	drop_pathr   )r   r   r   r   r   r   r{   r   rK   r   r   r   r   r   mid_chsrv   r   s                   rG   r   zBasicBlock.__init__   sv     	j$((***$799 <!788J00'Hf
 
 

  --fg{SYdlmndo-pp%._&+2EBKMMM6;;W^K_K_	--Wka[JRW . 
 
 /8h6;;NTZT_T_`gThTh5Cb5H5H.111bkmm$.L2;===FJJtJ4L4LrF   rT   c                    |rP| j         It          | j        j        dd           .t          j                            | j        j        j                   | j        | j	        fD ]&}t          |d          r|                                 'd S Nweightreset_parametersr   getattrr   bnr@   initzeros_r   r   r   hasattrr   r   rT   r   s      rG   init_weightszBasicBlock.init_weights       	5dm7GDNDUW_ae<f<f<rGNN4>,3444Y/ 	( 	(Dt/00 (%%'''	( 	(rF   c                 @   |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        ||                     |          z   }|                     |          S r`   )r   r   r   r   r   r   r   r   r   r   s      rG   r   zBasicBlock.forward      NN1IIaLLNN1NN1NN1=$DMM(+++Axx{{rF   )r   r"   r   Nr5   r   TFNNr   F)r<   r=   r>   __doc__rC   r   r
   rD   r?   rg   r   r   r   r   r   r   r   s   @rG   r   r      s?          !(.(,"%#"$"#'$&"M "M"M "M 	"M
 "M CHo"M !"M  "M "M "M "M "M !"M ""M "M "M "M "M "MH( (4 ( ( ( (	 	 	 	 	 	 	rF   r   c                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 dd
ededededeeef         dedee         dede	de	de	de	de
dedef fdZdde	fdZd Z xZS )BottleneckBlockz4 ResNet-like Bottleneck Block - 1x1 - kxk - 1x1
    r   r"   r   r5   Nr   Fr   r   r   r   r   r   r   r{   rK   r   r   
extra_conv	bottle_inr   r   r   c           	         t          t          |                                            |pt                      }t	          |r|n||z            }t          ||          }t          |||||d|          | _        |                    ||d          | _	        |                    |||||d         ||          | _
        |r&|                    ||||d         |          | _        nt          j                    | _        |	s|j        t          j                    n|                    |          | _        |                    ||dd          | _        |	r|j        t          j                    n|                    |          | _        |dk    rt#          |          nt          j                    | _        |
rt          j                    n|                    d	
          | _        d S )NFr   r"   r   r   r   rv   r   )r   rv   r   r   Tr   )r   r   r   r   r    r   r   r   r   	conv1_1x1r   
conv2b_kxkr@   r   r   	conv3_1x1r   r   r   r   )r   r   r   r   r   r   r   r{   rK   r   r   r   r   r   r   r   r   rv   r   s                     rG   r   zBottleneckBlock.__init__  s   $ 	ot$$--///$799 I"B&&7l!RSSJ00'Hf
 
 

  --fgqAA--WkHQK: . 
 
  	,$22+F 3 T TDOO !kmmDO%._&+2EBKMMM6;;W^K_K_	--gwU-SS.7h6;;NTZT_T_`gThTh5Cb5H5H.111bkmm$.L2;===FJJtJ4L4LrF   rT   c                    |rP| j         It          | j        j        dd           .t          j                            | j        j        j                   | j        | j	        fD ]&}t          |d          r|                                 'd S r   )r   r   r  r   r@   r   r   r   r   r   r   r   r   s      rG   r   zBottleneckBlock.init_weightsK  r   rF   c                    |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        ||                     |          z   }|                     |          S r`   )	r  r   r  r   r  r   r   r   r   r   s      rG   r   zBottleneckBlock.forwardR  s    NN1NN1OOAIIaLLNN1NN1NN1=$DMM(+++Axx{{rF   )r   r"   r   r5   Nr   FFFFNNr   r   r<   r=   r>   r   rC   r   rD   r
   r?   rg   r   r   r   r   r   r   r   s   @rG   r   r     s[          !(."$(,##$$#"#'$&!*M *M*M *M 	*M
 *M CHo*M  *M !*M *M *M *M *M *M *M !*M  "!*M *M *M *M *M *MX( (4 ( ( ( (      rF   r   c                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 ddededededeeef         dedee         dede	de	de
dedef fdZdde	fdZd Z xZS )	DarkBlocka
   DarkNet-like (1x1 + 3x3 w/ stride) block

    The GE-Net impl included a 1x1 + 3x3 block in their search space. It was not used in the feature models.
    This block is pretty much a DarkNet block (also DenseNet) hence the name. Neither DarkNet or DenseNet
    uses strides within the block (external 3x3 or maxpool downsampling is done in front of the block repeats).

    If one does want to use a lot of these blocks w/ stride, I'd recommend using the EdgeBlock (3x3 /w stride + 1x1)
    for more optimal compute.
    r   r"   r   r5   Nr   TFr   r   r   r   r   r   r   r{   rK   r   r   r   r   r   c           
      
   t          t          |                                            |pt                      }t	          ||z            }t          ||          }t          |||||d|          | _        |                    ||d          | _	        |	s|j
        t          j                    n|
                    |          | _
        |                    |||||d         ||d          | _        |	r|j
        t          j                    n|
                    |          | _        |dk    rt          |          nt          j                    | _        |
rt          j                    n|                    d          | _        d S )	NFr   r"   r   r   r   rv   r   r   r   Tr   )r   r  r   r   r    r   r   r   r   r  r   r@   r   r   r   r   r   r   r   r   r   r   r   r   r   r{   rK   r   r   r   r   r   r   rv   r   s                   rG   r   zDarkBlock.__init__k  si     	i'')))$799 <!788J00'Hf
 
 

  --fgqAA%._&+2EBKMMM6;;W^K_K_	--WkHQK:af . 
 
 /8h6;;NTZT_T_`gThTh5Cb5H5H.111bkmm$.L2;===FJJtJ4L4LrF   rT   c                    |rP| j         It          | j        j        dd           .t          j                            | j        j        j                   | j        | j	        fD ]&}t          |d          r|                                 'd S r   r   r   s      rG   r   zDarkBlock.init_weights  r   rF   c                 @   |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        ||                     |          z   }|                     |          S r`   )r  r   r   r   r   r   r   r   s      rG   r   zDarkBlock.forward  r   rF   )r   r"   r   r5   Nr   TFNNr   r   r  r   s   @rG   r  r  `  s?          !(."%(,#"$"#'$&"M "M"M "M 	"M
 "M CHo"M  "M !"M "M "M "M "M !"M ""M "M "M "M "M "MH( (4 ( ( ( (	 	 	 	 	 	 	rF   r  c                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 dd
ededededeeef         dedee         dede	de	de
dedef fdZdde	fdZd Z xZS )	EdgeBlocka   EdgeResidual-like (3x3 + 1x1) block

    A two layer block like DarkBlock, but with the order of the 3x3 and 1x1 convs reversed.
    Very similar to the EfficientNet Edge-Residual block but this block it ends with activations, is
    intended to be used with either expansion or bottleneck contraction, and can use DW/group/non-grouped convs.

    FIXME is there a more common 3x3 + 1x1 conv block to name this after?
    r   r"   r   r5   Nr   Fr   r   r   r   r   r   r   r{   rK   r   r   r   r   r   c           	         t          t          |                                            |pt                      }t	          ||z            }t          ||          }t          |||||d|          | _        |                    |||||d         ||          | _	        |	s|j
        t          j                    n|
                    |          | _
        |                    ||dd          | _        |	r|j
        t          j                    n|
                    |          | _        |dk    rt          |          nt          j                    | _        |
rt          j                    n|                    d	          | _        d S )
NFr   r   r   r"   r   r   Tr   )r   r  r   r   r    r   r   r   r   r   r   r@   r   	conv2_1x1r   r   r   r   r  s                   rG   r   zEdgeBlock.__init__  sk     	i'')))$799 <!788J00'Hf
 
 
  --G[HQK: . 
 
 &/_&+2EBKMMM6;;W^K_K_	--gwU-SS.7h6;;NTZT_T_`gThTh5Cb5H5H.111bkmm$.L2;===FJJtJ4L4LrF   rT   c                    |rP| j         It          | j        j        dd           .t          j                            | j        j        j                   | j        | j	        fD ]&}t          |d          r|                                 'd S r   )r   r   r  r   r@   r   r   r   r   r   r   r   r   s      rG   r   zEdgeBlock.init_weights  r   rF   c                 @   |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        ||                     |          z   }|                     |          S r`   )r   r   r  r   r   r   r   r   s      rG   r   zEdgeBlock.forward  r   rF   )r   r"   r   r5   Nr   FFNNr   r   r  r   s   @rG   r  r    s?          !(."%(,##$"#'$&!M !M!M !M 	!M
 !M CHo!M  !M !!M !M !M !M !M !!M "!M !M !M !M !M !MF( (4 ( ( ( (	 	 	 	 	 	 	rF   r  c                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 dd
ededededeeef         dedee         dede	de
dedef fdZddefdZd Zd Zdeej        ej        f         fdZdeej        ej        f         fdZ xZS ) RepVggBlockzQ RepVGG Block.

    Adapted from impl at https://github.com/DingXiaoH/RepVGG
    r   r"   r   r5   NrZ   r   Fr   r   r   r   r   r   r{   rK   r   r   r   inference_modec           
          t          t          |                                            t          ||          x| _        }|	pt                      }	|r!t          j        ||||||d          | _        nd | _        ||k    o|dk    o|d         |d         k    }|r|		                    |d          nd | _
        |	                    |||||d         ||
d          | _        |	                    ||d||d          | _        |d	k    r|rt          |          nt          j                    | _        |	j        t          j                    n|	                    |          | _        |	                    d
          | _        d S )NTin_channelsout_channelsr   r   r   rv   biasr"   r   Fr   r
  )r   rv   r   r   r   )r   r  r   r   rv   r   r@   Conv2dreparam_convr   identityr   conv_kxkconv_1x1r   r   r   r   r   )r   r   r   r   r   r   r   r{   rK   r   r   r   r  rv   	use_identr   s                  rG   r   zRepVggBlock.__init__  s    	k4  ))+++)*f===f$799 	n "	"$'!! ! !D !%D')XfkXhqkXVW[>XIIR\FOOGuOEEEX\DM"00Fzej 1  DM #00!F[amr0ssDM9G"9L9LQZ9LXn555`b`k`m`mDN%+[%8BKMMMfkk'>R>R	::d:++rF   rT   c                 `   |                                  D ]h}t          |t          j                  rLt          j                            |j        dd           t          j                            |j        dd           it          | j	        d          r| j	        
                                 d S d S )Ng?r   r   )modulesr   r@   BatchNorm2dr   normal_r   r  r   r   r   )r   rT   ms      rG   r   zRepVggBlock.init_weights  s     	/ 	/A!R^,, /"b1112...49011 	)I&&(((((	) 	)rF   c                    | j         ;|                     |                     |                      |                              S | j        ,|                     |          |                     |          z   }nZ|                     |          }|                     |          |                     |          z   }|                     |          }||z  }|                     |          }|                     |          S r`   )r  r   r   r  r   r  r   )r   r   r  s      rG   r   zRepVggBlock.forward  s    (88DIId&7&7&:&:;;<<<= a  4==#3#33AA}}Q''Ha  4==#3#33Aq!!AMAIIaLLxx{{rF   c           
         | j         dS |                                 \  }}t          j        | j        j        j        | j        j        j        | j        j        j        | j        j        j	        | j        j        j
        | j        j        j        | j        j        j        d          | _         || j         j        _        || j         j        _        |                                 D ]\  }}d|v r
|                                 |                     d           |                     d           |                     d           |                     d           dS )	   Following works like `RepVGG: Making VGG-style ConvNets Great Again` -
        https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched
        architecture used at training time to obtain a plain CNN-like structure
        for inference.
        NTr  r  r   r   paddingr   rv   r  r  r  r   r  r   r  _get_kernel_biasr@   r  r  r   r  r  r   r   r+  r   rv   r   datar  named_parametersdetach___delattr__r   kernelr  nameparas        rG   reparameterizezRepVggBlock.reparameterize*  sB    (F,,..I*6+8*6=%,M&.]'0=%,	
 	
 	
 )/ %&*# //11 	 	JD$%%LLNNNN$$$$$$$$$%%%%%rF   r   c                    d}d}| j         `|                     | j                   \  }}| j        j        j        d         dz  }t
          j        j                            |||||g          }d}d}| j	        |                     | j	                  \  }}|                     | j                  \  }}||z   |z   }||z   |z   }	||	fS z Method to obtain re-parameterized kernel and bias.
        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L83
        r   Nr2   )
r   _fuse_bn_tensorr  r   r   torchr@   
functionalpadr  )
r   
kernel_1x1bias_1x1r<  kernel_identitybias_identitykernel_conv	bias_convkernel_final
bias_finals
             rG   r-  zRepVggBlock._get_kernel_biasK  s    
 
=$#'#7#7#F#F J-$03q8C,00c3S=QRRJ =$-1-A-A$--P-P*O] "&!5!5dm!D!DY"Z//A)M9
Z''rF   c                    t          |t                    rI|j        j        }|j        j        }|j        j        }|j        j        }|j        j        }|j        j        }nt          |t          j
                  sJ t          | d          s| j        j        j        }|| j        z  }	| j        j        j        }
t!          j        | j        j        j                  }t%          |          D ] }d||||	z  |
d         dz  |
d         dz  f<   !|| _        | j        }|j        }|j        }|j        }|j        }|j        }||z                                   }||z                      dddd          }||z  |||z  |z  z
  fS ) Method to fuse batchnorm layer with preceeding conv layer.
        Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L95
        	id_tensorr"   r   r2   r   r   r   r   r   running_meanrunning_varr  epsr@   r$  r   r  r  rv   r   r:  
zeros_liker   rG  sqrtreshaper   branchr3  rJ  rK  gammabetarL  r   	input_dimr   kernel_valuer   stdts                  rG   r9  zRepVggBlock._fuse_bn_tensore  s    fk** 	['F!91L )/KI$E9>D)-CCfbn555554-- .+7"dk1	"m0<$/0B0IJJv a aA_`LA	M;q>Q3FTUZ[H[![\\!-^F!.L ,KME;D*CS &&((S[!!"aA..z4,"6"<<<<rF   )
r   r"   r   r5   NrZ   NNr   Fr   )r<   r=   r>   r   rC   r   rD   r
   r?   r   r   rg   r   r   r   r6  r:  Tensorr-  r9  r   r   s   @rG   r  r    s          !(."%(, "#'$&#(), ),), ), 	),
 ), CHo),  ), !), ), ), !), "), !), ), ), ), ), ),V) )4 ) ) ) )  & & &B(%el(B"C ( ( ( (4=u|U\/I)J = = = = = = = =rF   r  c                       e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 dd
ededededeeef         dedee         dede	dede
dededdf fdZdej        dej        fdZd Zdeej        ej        f         fdZdeej        ej        f         fdZ xZS )MobileOneBlocka8   MobileOne building block.

        This block has a multi-branched architecture at train-time
        and plain-CNN style architecture at inference time
        For more details, please refer to our paper:
        `An Improved One millisecond Mobile Backbone` -
        https://arxiv.org/pdf/2206.04040.pdf
    r   r"   r   r5   NrZ   Fr   r   r   r   r   r   r   r{   rK   r  r   r   r   r   r   c                    t          t          |                                            |
| _        t	          ||          x| _        }|pt                      }|	r"t          j        ||||||d          | _	        nd| _	        ||k    o|dk    o|d         |d         k    }|r|
                    |d          nd| _        g }t          | j                  D ]0}|                    |                    |||||d                     1t          j        |          | _        d| _        |dk    r |                    ||d||d          | _        |d	k    r|rt%          |          nt          j                    | _        |j        t          j                    n|                    |          | _        |                    d
          | _        dS )z, Construct a MobileOneBlock module.
        Tr  Nr"   r   Fr   )r   r   rv   r   r   r   )r   rZ  r   r   r   rv   r   r@   r  r  r   r  r   appendr   
ModuleListr  
conv_scaler   r   r   r   r   )r   r   r   r   r   r   r   r{   rK   r  r   r   r   r   rv   r!  convsr   r   s                     rG   r   zMobileOneBlock.__init__  s   $ 	nd##,,...!2)*f===f$799 	n "	"$'!! ! !D !%D ')XfkXhqkXVW[>XIIR\FOOGuOEEEX\DM E4122 D DV11G!&E 2 C C D D D D M%00DM #DOQ"("6"6G!&E #7 #C #C :H"9L9LQZ9LXn555`b`k`m`mDN%+[%8BKMMMfkk'>R>R	::d:++rF   r   c                    | j         ;|                     |                     |                      |                              S d}| j        |                     |          }d}| j        |                     |          }|}| j        D ]}| ||          z  }|                     |          }||z  }|                     |                     |                    S )z Apply forward pass. Nr   )r  r   r   r  r^  r  r   )r   r   identity_out	scale_outoutcks         rG   r   zMobileOneBlock.forward  s     (88DIId&7&7&:&:;;<<< =$==++L 	?&**I - 	 	B22a55LCCnnS!!|xx		#'''rF   c           
          | j         dS |                                 \  }}t          j        | j        d         j        j        | j        d         j        j        | j        d         j        j        | j        d         j        j	        | j        d         j        j
        | j        d         j        j        | j        d         j        j        d          | _         || j         j        _        || j         j        _        |                                 D ]\  }}d|v r
|                                 |                     d           |                     d           |                     d           |                     d	           dS )
r)  Nr   Tr*  r  r  r^  r  r   r,  r2  s        rG   r6  zMobileOneBlock.reparameterize  se    (F,,..Ia(-9q).;a(-9=#(/M!$)1]1%*3=#(/   )/ %&*# //11 	 	JD$%%LLNNNN$$$&&&$$$%%%%%rF   c                    d}d}| j         f|                     | j                   \  }}| j        d         j        j        d         dz  }t
          j        j                            |||||g          }d}d}| j	        |                     | j	                  \  }}d}d}t          | j                  D ]/}|                     | j        |                   \  }	}
||	z  }||
z  }0||z   |z   }||z   |z   }||fS r8  )r^  r9  r  r   r   r:  r@   r;  r<  r  r   r   )r   kernel_scale
bias_scaler<  r?  r@  rA  rB  ix_kernel_biasrC  rD  s                rG   r-  zMobileOneBlock._get_kernel_bias  s!   
 
?&'+';';DO'L'L$L*-"'3A6!;C 8.22<#sCQTAUVVL =$-1-A-A$--P-P*O] 	.// 	 	B!11$-2CDDNGU7"KII"\1OC+m;
Z''rF   c                    t          |t                    rI|j        j        }|j        j        }|j        j        }|j        j        }|j        j        }|j        j        }nt          |t          j
                  sJ t          | d          s| j        d         j        j        }|| j        z  }	| j        d         j        j        }
t!          j        | j        d         j        j                  }t%          |          D ] }d||||	z  |
d         dz  |
d         dz  f<   !|| _        | j        }|j        }|j        }|j        }|j        }|j        }||z                                   }||z                      dddd          }||z  |||z  |z  z
  fS )rF  rG  r   r"   r2   rH  rI  rP  s                  rG   r9  zMobileOneBlock._fuse_bn_tensor"  s    fk** 	['F!91L )/KI$E9>D)-CCfbn555554-- .q).:"dk1	"mA.3?$/a0@0E0LMMv a aA_`LA	M;q>Q3FTUZ[H[![\\!-^F!.L ,KME;D*CS &&((S[!!"aA..z4,"6"<<<<rF   )r   r"   r   r5   NrZ   Fr"   NNr   )r<   r=   r>   r   rC   r   rD   r
   r?   rg   r   r   r   r:  rX  r   r6  r-  r9  r   r   s   @rG   rZ  rZ    s          !(."%(, #(%&"#'$&8, 8,8, 8, 	8,
 8, CHo8,  8, !8, 8, !8,  #8, 8, !8, "8, 
8, 8, 8, 8, 8, 8,t( (%, ( ( ( (4& & &@(%el(B"C ( ( ( (>=u|U\/I)J = = = = = = = =rF   rZ  c            !            e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddededededeeef         dedee         dede	de	de	de	deeeef                  de
dedef  fdZd de	fdZd Z xZS )!SelfAttnBlockzI ResNet-like Bottleneck Block - 1x1 - optional kxk - self attn - 1x1
    r   r"   r   r5   Nr   FTr   r   r   r   r   r   r   r{   rK   r   r   r   post_attn_na	feat_sizer   r   r   c           	      F   t          t          |                                            |J t          |r|n||z            }t	          ||          }t          |||||d|          | _        |                    ||d          | _        |	r*|                    |||||d         ||          | _	        d}nt          j                    | _	        |i nt          |          } |j        |fd|i|| _        |r|                    |          nt          j                    | _        |                    ||dd          | _        |d	k    rt#          |          nt          j                    | _        |
rt          j                    n|                    d
          | _        d S )NFr   r"   r   r   )rp  r   r   r   Tr   )r   rn  r   r    r   r   r   r   r  r   r@   r   rb   r   r   	post_attnr  r   r   r   )r   r   r   r   r   r   r   r{   rK   r   r   r   ro  rp  r   r   r   r   rv   
opt_kwargsr   s                       rG   r   zSelfAttnBlock.__init__F  s   & 	mT""++---!!! I"B&&7l!RSSJ00'Hf
 
 

  --fgqAA 	+#11+Fz 2  DN FF[]]DN$,RR$2K2K2K
))'OO&OJOO5AT111r{}}--gwU-SS5Cb5H5H.111bkmm$.L2;===FJJtJ4L4LrF   rT   c                 
   |rP| j         It          | j        j        dd           .t          j                            | j        j        j                   t          | j	        d          r| j	        
                                 d S d S r   )r   r   r  r   r@   r   r   r   r   r   r   )r   rT   s     rG   r   zSelfAttnBlock.init_weightst  s~     	5dm7GDNDUW_ae<f<f<rGNN4>,34444>#566 	.N++-----	. 	.rF   c                 j   |}|                      |          }|                     |          }|                     |          }|                     |          }|                     |          }|                     |          }| j        ||                     |          z   }|                     |          S r`   )r  r   r   rr  r  r   r   r   r   s      rG   r   zSelfAttnBlock.forwardz  s    NN1NN1NN1NN1NN1NN1=$DMM(+++Axx{{rF   )r   r"   r   r5   Nr   FFFTNNNr   r   r  r   s   @rG   rn  rn  B  sv          !(."$(,#$$#!%37"#'$&#,M ,M,M ,M 	,M
 ,M CHo,M  ,M !,M ,M ,M ,M ,M ,M  c3h0,M ,M  !!,M" "#,M ,M ,M ,M ,M ,M\. .4 . . . .
 
 
 
 
 
 
rF   rn  )basicbottledarkedgerx   r   r   r   block_fnc                     |t           | <   d S r`   )_block_registry)r   rz  s     rG   register_blockr}    s    "*OJrF   blockc                     t          | t          j        t          f          r | di |S | t          v sJ d|              t	          |          di |S )NzUnknown block type (rE   )r   r@   rA   r   r|  )r~  r   s     rG   r-   r-     sh    %")W-.. uvO###%CE%C%C###5!++F+++rF   c                        e Zd Z	 	 	 	 	 	 	 ddedeeee         eedf         f         d	ed
edededee         de	de
f fdZdeej        eej                 f         fdZ xZS )Stemr   ri   rN   N      ?r   r   .r   r   r   num_repnum_act	chs_decayr   c
           
         t                                                       |dv sJ |	pt                      }	t          t          t
          f          rt                    }}
n%fdt          |          D             d d d         }
|| _        g | _	        d}dgdg|dz
  z  z   }|dk    r|sd|d<   ||n|}dg||z
  z  d	g|z  z   }|}d}d}t          t          |
||                    D ]\  }\  }}}|r|	j        nt          }d
|dz    }|dk    r6|dk    r0|dz
  }| j	                            t          |||d                     |                     | |||||                     |}||z  }|}|r|                                }|dv sJ |}| j	                            t          |||d                     |dk    r)|                     dt%          j        d                     n|dk    r)|                     dt%          j        d                     n`d|v r,|                     dt%          j        ddd                     n0d|v r,|                     dt%          j        dddd                     |dz  }d}|dk    r|nd | _        | j	                            t          |||d                     ||k    sJ d S )N)r2   ri   c                 :    g | ]}t          |z  z            S rE   )round)ry   r   r  r   s     rG   r|   z!Stem.__init__.<locals>.<listcomp>  s*    PPPAg	Q677PPPrF   rH  rZ   r2   r"   ri   FTr   r   num_chs	reductionmodulestage)r   r   )maxrN   r   avgpoolmax2avg2r  r   r  r  r   )r   r   r+  r   )r   r   r+  r   )r   r   r   r   r   r}   r   r   r   feature_infor   r~   r   r   r\  rb   
add_modulelowerr@   	MaxPool2dr   last_feat_idx)r   r   r   r   r   r   r  r  r  r   rQ   	prev_featstem_stridesstem_norm_actsprev_chscurr_strider  r   chr3   nalayer_fn	conv_namer   s     `     `              rG   r   zStem.__init__  sQ    	$799ge}-- 	W'llGHHPPPPPwPPPQUQUSUQUVH	saSGaK00Q;;t; L$_'''Gg$56$'9II'HlN(S(STT 		" 		"NA{Ar/1Dv++}H&q1uI1uuQ !A!((h+^gop)q)q)qrrrOOIxx"+^_'`'`'`aaaH1K!II 	::<<DOOOOOM$$T(kZckl%m%m%mnnnv~~Q8888Q8888$1VW(X(X(XYYYY$1VWkp(q(q(qrrr1KI.;q.@.@]]d  h+V_gh!i!i!ijjjf$$$$$$rF   r   c                 v    d }t          |           D ]$\  }} ||          }| j        || j        k    r|}%||fS r`   )r   r  )r   r   intermediater   r&  s        rG   forward_intermediateszStem.forward_intermediates  sS    /3dOO 	! 	!DAq!A!-!t7I2I2I ,rF   )r   ri   rN   r   Nr  N)r<   r=   r>   rC   r   r   r   r?   r
   rD   r   r   r:  rX  r  r   r   s   @rG   r  r    s         !!%)""A% A%A% 3S	5c?:;A% 	A%
 A% A% A% c]A% A% A% A% A% A% A% A%F%hu|>T0T*U        rF   r  rZ   stemrM   	pool_typefeat_prefixc                    |pt                      }|dv sJ d|v rd|v rdnd }t          | |d|||          }nd|v r t          | d|z  d	z  |dz  |f||
          }nd|v rt          | |dd||          }nd|v rt          | |d|          }nd|v rt          | |dd|          }nd|v r2|rt          | |dd||          }n||                    | |dd          }nbt          |t          t          f          rt          | |d||
          }n1|rt          | |dd||          }n|                    | |dd          }t          |t                    rfd|j        D             }nt          |dd          g}||fS )N)	rZ   quadquad2tiereddeeprx   r   7x7rL   r  r  r2   ri   )r  r  r   r   r  r   r   )r   r   r  r5   )r  r  r   r   rx   )r   r   r   )r   r   r   r     r"   )r  r   r   r   c           
      h    g | ].}t          |d                     |d         g                    /S ).r  )r  )rb   join)ry   fr  s     rG   r|   z$create_byob_stem.<locals>.<listcomp>  s<    hhhQRQsxxak0J'K'KLLLhhhrF   r   r  )
r   r  r  rZ  r   r   r}   r   r  rb   )	r   r   rM   r  r  r   r  r  r  s	       `    rG   r,   r,     s"     wyyF[[[[[)++!!FGQiX^___	Y		FQ[A-w!|WEI^deee	9		FGQ#IV\]]]	)		671VDDD	)		fg1QvVVV	)		 	FAIfUUUDD''1'EEDDgt}-- 	J6JJJDD  JFGQ	RXYYY++FGQq+II$ YhhhhVZVghhhW+UVWWWXrF   r2   c                 B    | d nt          fd| D                       S )Nc                     g | ]}|z  S rE   rE   )ry   r3   r   s     rG   r|   z$reduce_feat_size.<locals>.<listcomp>  s    0P0P0Pf0P0P0PrF   )r}   )rp  r   s    `rG   reduce_feat_sizer    s/    $44%0P0P0P0Pi0P0P0P*Q*QQrF   c                     | | n|}|pi S )a2   Override model level attn/self-attn/block kwargs w/ block level

    NOTE: kwargs are NOT merged across levels, block_kwargs will fully replace model_kwargs
    for the block if set to anything that isn't None.

    i.e. an empty block_kwargs dict will remove kwargs set at model level for that block
    rE   )r;   model_kwargs
out_kwargss      rG   override_kwargsr    s     ".!9|JrF   r;   	block_cfg	model_cfgc                 N   | d         }|j         d u}|s|j        c|r
|j         sd }nFt          |j        |j                  }|j         p|j         }|t          t	          |          fi |nd }t          ||          }|j        d u}|s|j        c|r
|j        sd }nFt          |j        |j                  }	|j        p|j        }|t          t	          |          fi |	nd }t          ||          }|| d<   |                     t          |j	        |j	                             d S )Nr   )r   r   )
r7   r8   r  r   r   r   r9   r:   updater;   )
r;   r  r  	layer_fnsattn_setr7   r8   self_attn_setr9   r:   s
             rG   update_block_kwargsr  *  s{   X&I #4/H 	89(4 	jI0 	jJJ))*?AVWWK"-E1EJISI_*!5!5EEEEEeiJIJ777	 -T9M 
B	2> 	9!: 	9"OO.y/I9Keff'7T9;TO". &h&?&?TTCSTTT48 IAAA	&L 	(>	@VWWXXXXXrF   r   r   output_stride	stem_featrp  block_kwargs_fnc                 :   |pt                      }g }d | j        D             }d |D             }	d t          j        d|t	          |	                                        |	          D             }
d}|d         }|d         }|}g }t          |          D ]`\  }}|d         j        }|dk    r|r|                    |           ||k    r|dk    r||z  }d}||z  }|dv rdnd	}g }t          |          D ]\  }}t          |j
        | j        z            }|j        }t          |t                    r |||          }t          |||dk    r|nd||f||j        | j        |
|         |         |
	  	        }|j        dv r||d<    ||||            |t'          |j        fi |gz  }|}|}|dk    r|dk    rt)          ||          }|t+          j        | gz  }t          ||d| |dz             }b|                    |           t+          j        | ||fS )Nc                 ,    g | ]}t          |          S rE   )r   )ry   r3   s     rG   r|   z&create_byob_stages.<locals>.<listcomp>Z  s!    ;;;1#A&&;;;rF   c                 @    g | ]}t          d  |D                       S )c                     g | ]	}|j         
S rE   r   )ry   bcs     rG   r|   z1create_byob_stages.<locals>.<listcomp>.<listcomp>[  s    ,,,B24,,,rF   )sum)ry   	stage_bcss     rG   r|   z&create_byob_stages.<locals>.<listcomp>[  s/    JJJ)c,,),,,--JJJrF   c                 6    g | ]}|                                 S rE   )tolist)ry   r   s     rG   r|   z&create_byob_stages.<locals>.<listcomp>\  s     
\
\
\!188::
\
\
\rF   r   r"   r  r  )r"   r2   r2   )	r   r   r   r   r{   r   rK   r   r   r  rp  )r  r  stages.r  )r   rI   r:  linspacer  splitr   r3   r\  r    r1   rR   r4   r   r   rb   r6   rK   r/   r-   r  r@   
Sequential)r   r   r  r  rp  r   r  r  r   depthsdprr   
net_strider  r  stages	stage_idxstage_block_cfgsr   first_dilationrI   	block_idxr  r   r{   r;   s                             rG   create_byob_stagesr  N  s     wyyFL;;
;;;JJJzJJJF
\
\u~aVMMSSTZ[[
\
\
\CH;'J#HIF'0'<'< &t &t#	#!!$&Q;;9;	***&&6A::HFf
&&00a$-.>$?$? 	@ 	@ Iy$Y[33C%CDDG"J*h// <'Z;;
!*avvQ((3%&\>"9~i8
 
 
L ~//,5[)OLIMMMM|INCClCCDDF%NHzzi1nn,Y??	2=&)**ZH]R[H]H]enqrersss			"""=&!<::rF   Tallow_aac                    t          | j                  }t          | j        |          }| j        r%|r#t          t          | j        || j                  }nt          t          | j        |          }| j        r$t          t          | j                  fi | j	        nd }| j
        r$t          t          | j
                  fi | j        nd }t          |||||          }|S )N)rY   rW   )rY   rW   r[   )r   r   r   r   r   )r   rW   r   rY   r[   r   r   r7   r   r8   r9   r:   r   )r   r  r   r   r   r   r   r  s           rG   get_layer_fnsr    s    

&
&C!S^sKKKH
| W WRU`c`lmmmRUVVVCF>[78CN++??s???W[DRUReo!455NN9MNNNkoI]X3UYenoooHOrF   c                   R    e Zd ZdZ	 	 	 	 	 	 	 	 d)ded	ed
edee         dedeeee	eef         f                  de
de
def fdZej        j        d*d            Zej        j        d+d            Zej        j        dej        fd            Zd,d	edee         fdZ	 	 	 	 	 	 d-dej        deeeee         f                  dedededededeeej                 e	ej        eej                 f         f         fd Z	 	 	 d.deeee         f         d"ed#efd$Zd% Zd*d&efd'Zd( Z xZS )/r)   a#   'Bring-your-own-blocks' Net

    A flexible network backbone that allows building model stem + blocks via
    dataclass cfg definition w/ factory functions for module instantiation.

    Current assumption is that both stem and blocks are in conv-bn-act order (w/ block ending in act).
      r   NrP   r   Tr   num_classesin_chansglobal_poolr  img_size	drop_rater   rT   c
           
         t                                                       || _        || _        d| _        t          fi |
t          d          }t                    }j        r|
J d            |t          |          nd}g | _	        t          j        t          t          f          rfdj        D             }n;t          t          j        pj        d         j        j        z                      }t'          ||j        j        |          \  | _        }| j	                            |dd                    t1          ||d         d	         
          }t3          |||d         ||          \  | _        }}| j	                            |dd                    |d         d	         }|d         d         }j        rPt          t          j        j        z                      | _        |                    || j        d          | _        n|| _        t=          j                    | _        | xj	        tA          | j        |dtC          | j                            gz  c_	        d | j	        D             | _"        | j        | _#        j$        dv sJ j$        dk    rJ|d}tK          | j        |j#        |j&        j'        | j                  | _(        | j(        j)        | _#        nщj$        dk    rF|d}|dv sJ tU          | j        j#        |||| j        d          | _(        | j(        j+        | _#        nj$        dk    rF|d}|dv sJ tY          | j        j#        |||| j        d          | _(        | j(        j+        | _#        n/|d}j#        J t[          | j        ||| j                  | _(        || _.        t_          ta          tb          |	          |            dS )a_  
        Args:
            cfg: Model architecture configuration.
            num_classes: Number of classifier classes.
            in_chans: Number of input channels.
            global_pool: Global pooling type.
            output_stride: Output stride of network, one of (8, 16, 32).
            img_size: Image size for fixed image size models (i.e. self-attn).
            drop_rate: Classifier dropout rate.
            drop_path_rate: Stochastic depth drop-path rate.
            zero_init_last: Zero-init last weight of residual path.
            **kwargs: Extra kwargs overlayed onto cfg.
        F)r  Nz8img_size argument is required for fixed input size modelc                 X    g | ]&}t          t          |j        z                      'S rE   )rC   r  rR   )ry   r1   r   s     rG   r|   z$ByobNet.__init__.<locals>.<listcomp>  s1    OOOQE!c&6"67788OOOrF   r   )r   r   rM   r  r   rH  r  r  )r   rp  r  r"   
final_convr  c                     g | ]
}|d          S )r  rE   )ry   r  s     rG   r|   z$ByobNet.__init__.<locals>.<listcomp>  s    AAA!1W:AAArF   )rZ   r]   mlpattn_absattn_rotr  r   )hidden_sizer  rY   rW   r  r  token)rZ   r  T)	embed_dimout_featuresrp  r  r  qkv_separater  )r  r  ref_feat_sizer  r  r  )r  r  rT   )2r   r   r  r  grad_checkpointingr   r  rU   r!   r  r   rQ   r   r}   rC   r  rI   r1   rR   r,   rM   rO   r  extendr  r  r  rS   r   r  r@   r   rb   r   
stage_endsr\   r^   r   rY   rW   headr  r   r  r   r   r  r%   r   _init_weights)r   r   r  r  r  r  r  r  r   rT   r   stem_layersstage_layersrp  rQ   r  
stage_featr  r  r   s    `                 rG   r   zByobNet.__init__  sh   4 	&""'c$$V$$#C%888$S)) 	d'')c'''+3+?Ih'''T	clT5M22 	XOOOO#,OOOHH5#,"A#*Q-/SEU!UVVWWH/mm 
  
  
	9 	  3B3000$Yy}[7QRRR	-?bM.
 .
 .
*Z 	  CRC111rN;/	b>), 	, #E#*:S=M*M$N$N O OD*884CTVWXXDOO (D kmmDO*i\_`d`k\l\lmmmo 	oAAt/@AAA $ 1} QQQQQ=E!!"#-!0%>-.  DI %)I$9D!!]j(("%-////'!.(#%.!  DI %)I$7D!!]Z''"%-////*!.('%.!  DI %)I$7D!!"#'///&!%.	  DI ' 	GM.III4PPPPPrF   Fc                 8    t          d|rdndd fdg          }|S )Nz^stemz^stages\.(\d+)z^stages\.(\d+)\.(\d+))z^final_conv)i )r  rI   ra   )r   coarsematchers      rG   group_matcherzByobNet.group_matcher(  s<    &,J""2JDQ*
 
 
 rF   c                     || _         d S r`   )r  )r   enables     rG   set_grad_checkpointingzByobNet.set_grad_checkpointing3  s    "(rF   r   c                     | j         j        S r`   )r  fc)r   s    rG   get_classifierzByobNet.get_classifier7  s    y|rF   c                 J    || _         | j                            ||           d S r`   )r  r  reset)r   r  r  s      rG   reset_classifierzByobNet.reset_classifier;  s&    &	[11111rF   NCHWr   indicesnorm
stop_early
output_fmtintermediates_onlyexclude_final_convc                     |dv s
J d            g }t          t           j                  |          \  }	}
 fd|	D             }	 j        |
         }
d}t           j        d          r j                            |          \  }}n                     |          d}}||	v r|                    ||n|            j        d         }t          j        	                                s|s j
        }n j
        d|
         }|D ]H}|dz  } ||          }|s||k    r                     |          }||	v r|                    |           I|r|S |r||k    r                     |          }||fS )	a9   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
            exclude_final_conv: Exclude final_conv from last intermediate
        Returns:

        )r  zOutput shape must be NCHW.c                 *    g | ]}j         |         S rE   )r  )ry   r   r   s     rG   r|   z1ByobNet.forward_intermediates.<locals>.<listcomp>Y  s     AAAq*AAArF   r   r  NrH  r"   )r$   r   r  r   r  r  r\  r:  jitis_scriptingr  r  )r   r   r	  r
  r  r  r  r  intermediatestake_indices	max_indexfeat_idxx_interlast_idxr  r  s   `               rG   r  zByobNet.forward_intermediates?  s   . Y&&&(D&&&"6s4?7K7KW"U"UiAAAALAAAOI.	49566 	,88;;JAww1twA|##  go7CCC?2&9!!## 	-: 	-[FF[),F 	( 	(EMHaA% '(h*>*>OOA&&<''$$Q''' 	!   	#(h"6"6""A-rF   r"   
prune_norm
prune_headc                    t          t          | j                  |          \  }}| j        |         }| j        d|         | _        || j        d         k     rt	          j                    | _        |r|                     dd           |S )z@ Prune layers not required for specified intermediates.
        NrH  r   rZ   )r$   r   r  r  r@   r   r  r  )r   r	  r  r  r  r  s         rG   prune_intermediate_layersz!ByobNet.prune_intermediate_layersz  s     #7s4?7K7KW"U"UiOI.	k*9*-tr*** kmmDO 	)!!!R(((rF   c                     |                      |          }| j        r4t          j                                        st          | j        |          }n|                     |          }|                     |          }|S r`   )r  r  r:  r  r  r&   r  r  r   s     rG   forward_featureszByobNet.forward_features  si    IIaLL" 	59+A+A+C+C 	t{A..AAAAOOArF   
pre_logitsc                 ^    |r|                      ||          n|                      |          S )N)r  )r  )r   r   r  s      rG   forward_headzByobNet.forward_head  s-    6@Rtyyzy222diiPQllRrF   c                 Z    |                      |          }|                     |          }|S r`   )r  r!  r   s     rG   r   zByobNet.forward  s-    !!!$$a  rF   )r  r   NrP   Nr   r   Tr   Tr`   )NFFr  FF)r"   FT)r<   r=   r>   r   r*   rC   r
   r?   r   r   rD   rg   r   r:  r  ignorer  r  r@   rA   r  r  rX  r   r  r  r  r!  r   r   r   s   @rG   r)   r)     s          $)-!#>B!#%#'CQ CQCQ CQ 	CQ
 "#CQ CQ uS%S/%9:;CQ CQ "CQ !CQ CQ CQ CQ CQ CQJ Y    Y) ) ) ) Y	    2 2C 2hsm 2 2 2 2 8<$$',',9  9 |9  eCcN349  	9 
 9  9  !%9  !%9  
tEL!5tEL7I)I#JJ	K9  9  9  9 z ./$#	 3S	>*  	   $  S S$ S S S S      rF   r)   c                 H   t          | t          j                  r| j        d         | j        d         z  | j        z  }|| j        z  }| j        j                            dt          j
        d|z                       | j         | j        j                                         d S d S t          | t          j                  rVt          j                            | j        dd           | j        &t          j                            | j                   d S d S t          | t          j                  rJt          j                            | j                   t          j                            | j                   d S t%          | d          r|                     |           d S d S )	Nr   r"          @r   g{Gz?)meanrV  r   r  )r   r@   r  r   r  rv   r   r.  r%  mathrN  r  zero_Linearr   r   r$  ones_r   r   )r  r4  rT   fan_outs       rG   r  r    su   &")$$ ;$Q'&*<Q*??&BUUFM!""1dig&>&>???;"K""$$$$$ #"	FBI	&	& ;
CT:::;"GNN6;''''' #"	FBN	+	+ ;
fm$$$
v{#####		(	( ;>:::::; ;rF   gernet_lrv  rp   r5   )r/   r0   r1   r3   r4   r6      rw  rj   i  g      ?         @ri   rP   i 
  )rI   rQ   rO   rS   gernet_mgernet_s0   r   r    i0  rq      i  	repvgg_a0)r2   ri      r"   )      ?r8  r8        @)r0   rz   rx   )rI   rM   rQ   	repvgg_a1)r"   r"   r"   r9  ro   	repvgg_a2)      ?r<  r<  g      @	repvgg_b0)r5   r5   r5   r9  )rz   	repvgg_b1)r&  r&  r&        @repvgg_b1g4)rz   rv   	repvgg_b2)r9  r9  r9        @repvgg_b2g4	repvgg_b3)r0  r0  r0  rB  repvgg_b3g4repvgg_d2se)r   r7     r"   r   g      ?)rd_ratio
rd_divisor)rI   rM   rQ   r7   r8   	resnet51qrr   i   r  i   silu)rI   rQ   rM   rO   rS   rW   	resnet61qry  )r/   r0   r1   r3   r4   r6   r;   r  )r   )rI   rQ   rM   rO   rS   rW   r;   resnext26tsi   r  rN   )rI   rQ   rM   rO   rW   gcresnext26tsgca)rI   rQ   rM   rO   rW   r7   seresnext26tseca_resnext26tsecabat_resnext26tsbatr   )
block_size)rI   rQ   rM   rO   rW   r7   r8   
resnet32ts
resnet33tsi   gcresnet33ts)rI   rQ   rM   rO   rS   rW   r7   seresnet33tseca_resnet33tsgcresnet50t)r/   r0   r1   r3   r6   )rI   rQ   rM   rO   r7   gcresnext50tsregnetz_b16rk   `         )rH  )r   r   )	rI   rQ   rO   rK   rS   rW   r7   r8   r;   regnetz_c16regnetz_d32i   )
rI   rQ   rM   rO   rK   rS   rW   r7   r8   r;   
regnetz_d8
regnetz_e8regnetz_b16_evos)r{   )
rI   rQ   rO   rK   rS   rW   rY   r7   r8   r;   regnetz_c16_evosregnetz_d8_evosr  )rI   rQ   rM   rO   rK   rS   rW   rY   r7   r8   r;   mobileone_s0)r8  r5   r5   r&  )rz   r   r   mobileone_s1)r<  r<  r&  r9  mobileone_s2)r<  r&  r9  r?  mobileone_s3)r&  r9  r0  r?  mobileone_s4)r0        @rm  r?  )r   r   r/  r"   )rz   r   resnet50_clip)rP   rP   ro   r  r   r  )rI   rQ   rM   rO   rK   r[   r^   resnet101_clip   resnet50x4_clipr   g      ?)rI   rR   rQ   rM   rO   rK   r[   r^   resnet50x16_clip   r<  resnet50x64_clip   $   r&  resnet50_mlpr  )rI   rQ   rM   rO   rK   r[   r\   r^   test_byobnetr  rx  rG  rV   )rI   rQ   rK   rO   rW   r7   r8   )rn  ro  rq  rr  rt  r]   )r^   _gapvisual.
state_dictmodelprefixc                    t          |j        t          t          f          }dd l}d }d }i }|                                 D ]+\  }}	|                    |          s|                    | dd|          }|                    | dd|          }|                    | d||          }|                    | d	||          }|                    | d
          r|s|                    |d
z   d          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|                    dd          }|	||<   -|S )Nr   c                 <   t          |                     d                    dz
  }t          |                     d                    |                     d          t          |                     d                    }}}d| d| d}ddd	d
}||         |z   }||z   S )Nr"   r2   r   ri   r  r  z
conv1_1x1.z
conv2_kxk.z
conv3_1x1.)r"   r2   r   rC   group)r&  r  	layer_idx
layer_typelayer_id
prefix_strid_map
suffix_strs           rG   
_stage_subz(_convert_openai_clip.<locals>._stage_sub  s    

OOa'	*-aggajj//1771::s177ST::x:	7y779777
!l|DDH%
2
J&&rF   c                     t          |                     d                    dz
  }t          |                     d                    t          |                     d                    }}d| d| d|dk    rdnd	z   S )
Nr"   r2   r   r  r  z
.shortcut.r   z	conv.convzconv.bnr  )r&  r  r  r  s       rG   	_down_subz'_convert_openai_clip.<locals>._down_sub  sq    

OOa'	!!''!**oos1771::8	:::Y:::XYZ]]kk`ijjrF   zconv([0-9])zstem.conv\1.convz	bn([0-9])zstem.conv\1.bnz'layer([0-9])\.([0-9]+)\.([a-z]+)([0-9])z+layer([0-9])\.([0-9]+)\.downsample\.([0-9])attnpoolr  positional_embedding	pos_embedq_projqk_projkv_projvc_projproj)	r   r  r   r   reitems
startswithsubr   )
r{  r|  r}  model_has_attn_poolr  r  r  out_dictr  r  s
             rG   _convert_openai_clipr    s   
 %UZ2Do1VWWIII' ' 'k k k
 H  ""  1||F## 	FFv***,?CCFFv(((*;Q??FFvFFF
TUVVFFvJJJIWXYY<<6+++,, 	,& 		&:-v66A		0+>>A		(C((A		(C((A		(C((A		(F++AOrF   c                 .    d| v rt          | |          } | S )Nzvisual.conv1.weight)r  )r{  r|  s     rG   checkpoint_filter_fnr    s$     
**)*e<<
rF   c                 n    t          t          | |ft          |          t          t	          d          d|S )NT)flatten_sequential)r  pretrained_filter_fnfeature_cfg)r#   r)   
model_cfgsr  rb   )variant
pretrainedr   s      rG   _create_byobnetr  !  sH    *W%1D111	 
   rF   c                 4    | dddddt           t          ddd
|S )	Nr  r      r  r  r        ?bilinear	stem.convhead.fc
urlr  
input_size	pool_sizecrop_pctinterpolationr'  rV  
first_convr]   r   r   r  r   s     rG   _cfgr  +  s5    4}SYJ%.B!	 
  rF   c                 4    | dddddt           t          ddd
|S )	Nr  r   rq   rq   r   r   ?bicubiczstem.conv1.convr  r  r  r  s     rG   _cfgrr  5  s5    4}SY)%.B'y	 
  rF   zgernet_s.idstcv_in1kztimm/)	hf_hub_idzgernet_m.idstcv_in1kzgernet_l.idstcv_in1kr  r  )r  r  r  zrepvgg_a0.rvgg_in1k)zstem.conv_kxk.convzstem.conv_1x1.convmit)r  r  licensezrepvgg_a1.rvgg_in1kzrepvgg_a2.rvgg_in1kzrepvgg_b0.rvgg_in1kzrepvgg_b1.rvgg_in1kzrepvgg_b1g4.rvgg_in1kzrepvgg_b2.rvgg_in1kzrepvgg_b2g4.rvgg_in1kzrepvgg_b3.rvgg_in1kzrepvgg_b3g4.rvgg_in1kzrepvgg_d2se.rvgg_in1k)r   @  r  )r   r   )r  r  r  r  r  r  zresnet51q.ra2_in1kzkhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet51q_ra2-d47dcc76.pthz
stem.conv1)r   r`  r`  )r  r  r  r  r  test_input_sizetest_crop_pctzresnet61q.ra2_in1kzkhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/resnet61q_ra2-6afc536c.pth)r  r  r  r  zresnext26ts.ra2_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnext26ts_256_ra2-8bbd9106.pthzseresnext26ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/seresnext26ts_256-6f0d74a3.pthzgcresnext26ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnext26ts_256-e414378b.pthzeca_resnext26ts.ch_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/eca_resnext26ts_256-5a1d030f.pthzbat_resnext26ts.ch_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/bat_resnext26ts_256-fa6fd595.pth)r  r  min_input_sizezresnet32ts.ra2_in1kzqhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnet32ts_256-aacf5250.pthzresnet33ts.ra2_in1kzqhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/resnet33ts_256-e91b09a4.pthzgcresnet33ts.ra2_in1kzshttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnet33ts_256-0e0cd345.pthzseresnet33ts.ra2_in1kzshttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/seresnet33ts_256-f8ad44d9.pthzeca_resnet33ts.ra2_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/eca_resnet33ts_256-8f98face.pthzgcresnet50t.ra2_in1kzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnet50t_256-96374d1c.pthzgcresnext50ts.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/gcresnext50ts_256-3e0f515e.pthzregnetz_b16.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_b_raa-677d9606.pthr  )r  r  r  r  r  gGz?)
r  r  r  r'  rV  r  r  r  r  r  zregnetz_c16.ra3_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_c_rab2_256-a54bf36a.pth)r  r  r  r'  rV  r  r  r  zregnetz_d32.ra3_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_d_rab_256-b8073a89.pthgffffff?)r  r  r'  rV  r  r  zregnetz_d8.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_d8_bh-afc03c55.pth)r  r  r'  rV  r  r  r  zregnetz_e8.ra3_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-attn-weights/regnetz_e8_bh-aace8e6e.pthzregnetz_b16_evos.untrained)r  r'  rV  r  r  r  r  zregnetz_c16_evos.ch_in1kzuhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/regnetz_c16_evos_ch-d8311942.pth)r  r  r  r'  rV  r  r  zregnetz_d8_evos.ch_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/regnetz_d8_evos_ch-2bc12646.pthzmobileone_s0.apple_in1kr  )zstem.conv_kxk.0.convzstem.conv_scale.conv)r  r  r  zmobileone_s1.apple_in1kr  zmobileone_s2.apple_in1kzmobileone_s3.apple_in1kzmobileone_s4.apple_in1kzresnet50_clip.openaizopen_clip_pytorch_model.binz	head.proj)	r  hf_hub_filenamer  r'  rV  rU   r  r  r]   zresnet101_clip.openaizresnet50x4_clip.openai)	   r  zresnet50x16_clip.openaii   )r   r4  r4  )r_  r_  zresnet50x64_clip.openai)r     r  )r7  r7  zresnet50_clip_gap.openaiztimm/resnet50_clip.openai)r  r  r  r'  rV  r  r  zresnet101_clip_gap.openaiztimm/resnet101_clip.openaizresnet50x4_clip_gap.openaiztimm/resnet50x4_clip.openaizresnet50x16_clip_gap.openaiztimm/resnet50x16_clip.openaizresnet50x64_clip_gap.openaiztimm/resnet50x64_clip.openai)r  r  )r      r  )r/  r/  )r  r  r  r  r  )zresnet50_mlp.untrainedztest_byobnet.r160_in1kc                     t          dd| i|S )z GEResNet-Large (GENet-Large from official impl)
    `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
    r-  r  )r-  r  r  r   s     rG   r-  r-  ;	      
 GG*GGGGrF   c                     t          dd| i|S )z GEResNet-Medium (GENet-Normal from official impl)
    `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
    r1  r  )r1  r  r  s     rG   r1  r1  C	  r  rF   c                     t          dd| i|S )z EResNet-Small (GENet-Small from official impl)
    `Neural Architecture Design for GPU-Efficient Networks` - https://arxiv.org/abs/2006.14090
    r2  r  )r2  r  r  s     rG   r2  r2  K	  r  rF   c                     t          dd| i|S )z^ RepVGG-A0
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r6  r  )r6  r  r  s     rG   r6  r6  S	      
 HH:HHHHrF   c                     t          dd| i|S )z^ RepVGG-A1
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r:  r  )r:  r  r  s     rG   r:  r:  [	  r  rF   c                     t          dd| i|S )z^ RepVGG-A2
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r;  r  )r;  r  r  s     rG   r;  r;  c	  r  rF   c                     t          dd| i|S )z^ RepVGG-B0
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r=  r  )r=  r  r  s     rG   r=  r=  k	  r  rF   c                     t          dd| i|S )z^ RepVGG-B1
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r>  r  )r>  r  r  s     rG   r>  r>  s	  r  rF   c                     t          dd| i|S )z` RepVGG-B1g4
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    r@  r  )r@  r  r  s     rG   r@  r@  {	      
 JJZJ6JJJrF   c                     t          dd| i|S )z^ RepVGG-B2
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    rA  r  )rA  r  r  s     rG   rA  rA  	  r  rF   c                     t          dd| i|S )z` RepVGG-B2g4
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    rC  r  )rC  r  r  s     rG   rC  rC  	  r  rF   c                     t          dd| i|S )z^ RepVGG-B3
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    rD  r  )rD  r  r  s     rG   rD  rD  	  r  rF   c                     t          dd| i|S )z` RepVGG-B3g4
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    rE  r  )rE  r  r  s     rG   rE  rE  	  r  rF   c                     t          dd| i|S )z` RepVGG-D2se
    `Making VGG-style ConvNets Great Again` - https://arxiv.org/abs/2101.03697
    rF  r  )rF  r  r  s     rG   rF  rF  	  r  rF   c                     t          dd| i|S )
    rJ  r  )rJ  r  r  s     rG   rJ  rJ  	       HH:HHHHrF   c                     t          dd| i|S )r  rL  r  )rL  r  r  s     rG   rL  rL  	  r  rF   c                     t          dd| i|S )r  rM  r  )rM  r  r  s     rG   rM  rM  	       JJZJ6JJJrF   c                     t          dd| i|S )r  rN  r  )rN  r  r  s     rG   rN  rN  	       LLzLVLLLrF   c                     t          dd| i|S )r  rP  r  )rP  r  r  s     rG   rP  rP  	  r  rF   c                     t          dd| i|S )r  rQ  r  )rQ  r  r  s     rG   rQ  rQ  	       NNNvNNNrF   c                     t          dd| i|S )r  rS  r  )rS  r  r  s     rG   rS  rS  	  r  rF   c                     t          dd| i|S )r  rV  r  )rV  r  r  s     rG   rV  rV  	       IIJI&IIIrF   c                     t          dd| i|S )r  rW  r  )rW  r  r  s     rG   rW  rW  	  r  rF   c                     t          dd| i|S )r  rX  r  )rX  r  r  s     rG   rX  rX  	       KKjKFKKKrF   c                     t          dd| i|S )r  rY  r  )rY  r  r  s     rG   rY  rY  	  r  rF   c                     t          dd| i|S )r  rZ  r  )rZ  r  r  s     rG   rZ  rZ  	       MM
MfMMMrF   c                     t          dd| i|S )r  r[  r  )r[  r  r  s     rG   r[  r[  	  r  rF   c                     t          dd| i|S )r  r\  r  )r\  r  r  s     rG   r\  r\  
  r  rF   c                     t          dd| i|S )r  r]  r  )r]  r  r  s     rG   r]  r]  
  r  rF   c                     t          dd| i|S )r  ra  r  )ra  r  r  s     rG   ra  ra  
  r  rF   c                     t          dd| i|S )r  rb  r  )rb  r  r  s     rG   rb  rb  
  r  rF   c                     t          dd| i|S )r  rc  r  )rc  r  r  s     rG   rc  rc  "
  r  rF   c                     t          dd| i|S )r  rd  r  )rd  r  r  s     rG   rd  rd  )
  r  rF   c                     t          dd| i|S )r  re  r  )re  r  r  s     rG   re  re  0
       OO*OOOOrF   c                     t          dd| i|S )r  rf  r  )rf  r  r  s     rG   rf  rf  7
  r  rF   c                     t          dd| i|S )r  rg  r  )rg  r  r  s     rG   rg  rg  >
  r  rF   c                     t          dd| i|S )r  rh  r  )rh  r  r  s     rG   rh  rh  E
  r  rF   c                     t          dd| i|S )r  ri  r  )ri  r  r  s     rG   ri  ri  L
  r  rF   c                     t          dd| i|S )r  rj  r  )rj  r  r  s     rG   rj  rj  S
  r  rF   c                     t          dd| i|S )r  rk  r  )rk  r  r  s     rG   rk  rk  Z
  r  rF   c                     t          dd| i|S )r  rl  r  )rl  r  r  s     rG   rl  rl  a
  r  rF   c                     t          dd| i|S )z0 OpenAI Modified ResNet-50 CLIP image tower
    rn  r  )rn  r  r  s     rG   rn  rn  h
  r  rF   c                     t          dd| i|S )z1 OpenAI Modified ResNet-101 CLIP image tower
    ro  r  )ro  r  r  s     rG   ro  ro  o
  r  rF   c                     t          dd| i|S )z2 OpenAI Modified ResNet-50x4 CLIP image tower
    rq  r  )rq  r  r  s     rG   rq  rq  v
  r  rF   c                     t          dd| i|S )z3 OpenAI Modified ResNet-50x16 CLIP image tower
    rr  r  )rr  r  r  s     rG   rr  rr  }
  r  rF   c                     t          dd| i|S )z3 OpenAI Modified ResNet-50x64 CLIP image tower
    rt  r  )rt  r  r  s     rG   rt  rt  
  r  rF   c                     t          dd| i|S )zP OpenAI Modified ResNet-50 CLIP image tower w/ avg pool (no attention pool)
    resnet50_clip_gapr  )r  r  r  s     rG   r  r  
  s     PP:PPPPrF   c                     t          dd| i|S )zQ OpenAI Modified ResNet-101 CLIP image tower w/ avg pool (no attention pool)
    resnet101_clip_gapr  )r	  r  r  s     rG   r	  r	  
  s     QQJQ&QQQrF   c                     t          dd| i|S )zR OpenAI Modified ResNet-50x4 CLIP image tower w/ avg pool (no attention pool)
    resnet50x4_clip_gapr  )r  r  r  s     rG   r  r  
  s     RRZR6RRRrF   c                     t          dd| i|S )zS OpenAI Modified ResNet-50x16 CLIP image tower w/ avg pool (no attention pool)
    resnet50x16_clip_gapr  )r  r  r  s     rG   r  r  
       SSjSFSSSrF   c                     t          dd| i|S )zS OpenAI Modified ResNet-50x64 CLIP image tower w/ avg pool (no attention pool)
    resnet50x64_clip_gapr  )r  r  r  s     rG   r  r  
  r  rF   c                     t          dd| i|S )r  rw  r  )rw  r  r  s     rG   rw  rw  
  r  rF   c                     t          dd| i|S )z- Minimal test ResNet (BYOB based) model.
    rx  r  )rx  r  r  s     rG   rx  rx  
  r  rF   )rh   rl   r   )r   rl   rE   r"   )r"   F)rZ   rZ   r  N)r2   r#  )rZ   F)rz  r   )rZ   )r   r(  dataclassesr   r   r   	functoolsr   typingr   r   r	   r
   r   r   r   r   r:  torch.nnr@   	timm.datar   r   r   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   _builderr#   	_featuresr$   _manipulater%   r&   	_registryr'   r(   __all__r+   r*   r   r   r?   rC   rg   r   r   r   r   rA   r   r   r   r   r  r  r  rZ  rn  rb   r|  r}  r-   r  r  r,   r  r  r  rD   r  r  r)   r  r  r  rX  r  r  r  r  r  default_cfgsr-  r1  r2  r6  r:  r;  r=  r>  r@  rA  rC  rD  rE  rF  rJ  rL  rM  rN  rP  rQ  rS  rV  rW  rX  rY  rZ  r[  r\  r]  ra  rb  rc  rd  re  rf  rg  rh  ri  rj  rk  rl  rn  ro  rq  rr  rt  r  r	  r  r  r  rw  rx  rE   rF   rG   <module>r     s`6   <  1 1 1 1 1 1 1 1 1 1       N N N N N N N N N N N N N N N N N N N N        d d d d d d d d d d d d                                 
 + * * * * * + + + + + + 4 4 4 4 4 4 4 4 < < < < < < < <
W
W
W 2 2 2 2 2 2 2 2  I I I I I I I I:      . () S#XS$s)^$ 
 ;   *k8K;P.P(Q VZ[fVg    & & & ) ) ) ) ) ) ) )' ' ' ' 'BI ' ' '2  	
 S/    *8 8 8 8 8 8 8 8vB B B B Bbi B B BJ? ? ? ? ?	 ? ? ?D= = = = =	 = = =@`= `= `= `= `=") `= `= `=Fz= z= z= z= z=RY z= z= z=zB B B B BBI B B BJ $
		  +c +RY + + + +,c29n- , , , ,K K K K K2= K K Kb !, ,,, , 	,
 , , , , ,^R R R R	 	 	!Yd38n !Y !YYd !Y !Y !Y !YR $($(.A=; =;=;=; =; S>	=;
 C==; !=; "(+=; =; =; =;@
 
{ 
d 
 
 
 
@ @ @ @ @bi @ @ @F; ; ; ;$ T x x x[KWSA!CCCKWSA!CCCKXcQ1GGGKXcQ1DDDKXcQ1DDD
    x [KWSA!CCCKWSA!CCCKXcQ1GGGKXcQ1DDDKXcQ1DDD
    x2 [KWR1rBBBKWR1rBBBKXcQ1GGGKXcQ1DDDKXcQ1DDD
    3xL k}}1HIII   MxV k}}@@@   Wx` k}}1FGGG   axj k} 1222   kxt k} 0111   ux~ } 0;;;   xH k} 3444   IxR } 3A>>>   Sx\ k} 0111   ]xf } 0;;;   gxp }~2EFFFD&Q777   qxD kKXcQ2$GGGKXcQ2$GGGKXdaB4HHHKXdaA#FFF	
    Exd kKVqC1sQUQUQWQWXXXKXcQ2$GGGKXdaB4HHHKXdaA#FFF	
 TT***   exF KXcQ2$GGGKXcQ2$GGGKXdaB4HHHKXdaB4HHH	
    Gx^ +KXcQ2$GGGKXcQ2$GGGKXdaB4HHHKXdaB4HHH	
    _xx +KXcQ2$GGGKXcQ2$GGGKXdaB4HHHKXdaB4HHH	
    yxR  KKXcQ2$GGGKXcQ2$GGGKXdaB4HHHKXdaB4HHH	
    Sxl  KKXcQ2$GGGKXcQ2$GGGKXdaB4HHHKXdaB4HHH	
 DA&&&   mxL {KXcQ1FFFKXcQ1FFFKXdaA$GGGKXdaA$GGG	
    Mxj {KXcQ1FFFKXcQ1FFFKXdaA$GGGKXdaA$GGG	
    kxJ KXcQ1FFFKXcQ1FFFKXdaA$GGGKXdaA$GGG	
    Kxf KXcQ1FFFKXcQ1FFFKXdaA$GGGKXdaA$GGG	
    gxB ;KXcQ1FFFKXcQ1FFFKXdaA$GGGKXdaA$GGG	
    Cx` KXcQ4@@@KXcQ4@@@KXdaDAAAKXdaDAAA	
    axz +KXcQ2$GGGKXcQ2$GGGKXdaB4HHHKXdaB4HHH	
    {xX	 KXbA"CCCKXbA"CCCKXsaB1EEEKXcQ2!DDD	
 D$'''TDT:::   Y	xx	 KXbA"CCCKXbA"CCCKXsaB1EEEKXcQ2!DDD	
 D$'''TDT:::   y	xX
 KXbA"CCCKXcQ2!DDDKXsaB1EEEKXcQ2!DDD	
 D$'''TDT:::   Y
xz
 {KXbA!BBBKXcQ1CCCKXsaA!DDDKXcQ1CCC	
 D$'''TDT:::   {
x\ {KXbA!BBBKXcQ1CCCKXsaA!DDDKXcQ1CCC	
 D$'''TDT:::   ]xB ![KXbA"CCCKXbA"CCCKXsaB1EEEKXcQ2!DDD	
 7<B777D$'''TDT:::   Cxd ![KXbA"CCCKXbA"CCCKXsaB1EEEKXcQ2!DDD	
 7<B777D$'''TDT:::   exF  KKXbA!BBBKXcQ1CCCKXsaA!DDDKXcQ1CCC	
 7<B777D$'''TDT:::!   Gxl "6!LLL   mxv "6777   wx@ "6777   AxJ "6777   KxT "6,OOO   Ux` +KXcQ4@@@KXcQ4@@@KXdaDAAAKXdaDAAA	
    ax| ;KXcQ4@@@KXcQ4@@@KXtqTBBBKXdaDAAA	
    }xX  KKXcQ4@@@KXcQ4@@@KXtqTBBBKXdaDAAA	
    Yxv ![KXcQ4@@@KXcQ4@@@KXtqTBBBKXdaDAAA	
    wxT ![KXcQ4@@@KXsaDAAAKXtqTBBBKXtqTBBB	
    Uxt KXcQ4@@@KXcQ4@@@KXdaDAAAKXdaDAAA	
    uxT KVqB!cBBBKVqB!cBBBKWSA"FFFKXcQ2$GGG	
 D$'''   Ux
r 
h L LA$WZ]lKKKJq6z  ( (el*+(( ( 
#u|
	( ( ( (Vel*+             %$ y&DD7333y& DD7333y& DD7}X^___	y& 44?P P Py& 44?P P Py& 44?P P Py&  44?P P P!y&& 44?P P P'y&, TT?P P P-y&2 44?P P P3y&8 TT?P P P9y&> 44?P P P?y&D TT?P P PEy&J TT? Hs  Ky&X $$yMV%S	: : :Yy&b %%y%S: : :cy&n EE E%S: : :oy& y&v UU C%S: : :wy&~ UU C%S: : :y&F uu E%S :  :  :Gy&N uu E$ &  &  &Oy&Z 55%S: : :[y&b 55%S: : :cy&j UU B%S: : :ky&r UU B%S: : :sy&z uu D%S :  :  :{y&D EE A%S: : :Ey&N UU C%S: : :Oy&Z EE~_/ FTS`ps	u u u[y&d EE D_/}C	I I Iey&n EE C/DR_a a aoy&v 55~/DR_ort t twy&~ 55~/DR_ort t ty&H !%%_/ FTS`#b #b #bIy& y& y&N  D_/}	!6 !6 !6Oy&X uu C/DR_or t  t  tYy&b ttC     cy&l ttC     my&v ttC     wy&@ ttC     Ay&J ttC     Ky&X EE5/_-6  Yy&f UU5.O-6  gy&t ee5.O-6  uy&B uu5.O-8     Cy&P uu5/_-8     Qy&b -5,/ F	! ! !cy&n  .5,/ F	" " "oy&z !%%/5,/ F	# # #{y&F "5505,/ H	$ $ $Gy&R "5505,/ H	$ $ $Sy& y&` $e F   $e 46  iy& y& y& y yx H HG H H H H H HG H H H H H HG H H H H I IW I I I I I IW I I I I I IW I I I I I IW I I I I I IW I I I I K Kw K K K K I IW I I I I K Kw K K K K I IW I I I I K Kw K K K K K Kw K K K K I IW I I I I I IW I I I I K Kw K K K K M M M M M M M M M M M M O O7 O O O O O O7 O O O O J Jg J J J J J Jg J J J J L L L L L L L L L L L L N N' N N N N K Kw K K K K M M M M M M K Kw K K K K K Kw K K K K K Kw K K K K J Jg J J J J J Jg J J J J P PG P P P P P PG P P P P O O7 O O O O L L L L L L L L L L L L L L L L L L L L L L L L L L L L L L M M M M M M N N' N N N N O O7 O O O O P PG P P P P P PG P P P P Q QW Q Q Q Q R Rg R R R R S Sw S S S S T T T T T T T T T T T T L L L L L L L L L L L L L LrF   