
    Ngks                        d Z ddlZddlmZmZmZmZmZ ddlZddl	m
Z
 ddlm
c mZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZmZmZmZ dd	lm Z  dd
l!m"Z" ddl#m$Z$m%Z% dgZ&dee'e'f         dej(        fdZ) G d de
j*                  Z+ G d de
j*                  Z, G d de
j*                  Z- G d de
j*                  Z.d2dZ/ e$ e/d           e/ddd           e/dd           e/d           e/ddd           e/dd d           e/dd           e/dee!           e/dee!           e/ddee"           e/dd#ee$           e/dd#ee$           e/ddee"          d%          Z0d3d(Z1d4d*Z2e%d4de.fd+            Z3e%d4de.fd,            Z4e%d4de.fd-            Z5e%d4de.fd.            Z6e%d4de.fd/            Z7e%d4de.fd0            Z8e%d4de.fd1            Z9dS )5a   BEiT: BERT Pre-Training of Image Transformers (https://arxiv.org/abs/2106.08254)

Model from official source: https://github.com/microsoft/unilm/tree/master/beit

@inproceedings{beit,
title={{BEiT}: {BERT} Pre-Training of Image Transformers},
author={Hangbo Bao and Li Dong and Songhao Piao and Furu Wei},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=p-BhZSz59o4}
}

BEiT-v2 from https://github.com/microsoft/unilm/tree/master/beit2

@article{beitv2,
title={{BEiT v2}: Masked Image Modeling with Vector-Quantized Visual Tokenizers},
author={Zhiliang Peng and Li Dong and Hangbo Bao and Qixiang Ye and Furu Wei},
year={2022},
eprint={2208.06366},
archivePrefix={arXiv},
primaryClass={cs.CV}
}

At this point only the 1k fine-tuned classification weights and model configs have been added,
see original source above for pre-training models and procedure.

Modifications by / Copyright 2021 Ross Wightman, original copyrights below
    N)CallableListOptionalTupleUnion)
checkpoint)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpSwiGLU	LayerNormDropPathtrunc_normal_use_fused_attn)resample_patch_embedresample_abs_pos_embedresize_rel_pos_bias_tablendgrid   )build_model_with_cfg)feature_take_indices)generate_default_cfgsregister_modelBeitwindow_sizereturnc           	      L   d| d         z  dz
  d| d         z  dz
  z  dz   }| d         | d         z  }t          j        t          t          j        | d                   t          j        | d                                       }t          j        |d          }|d d d d d f         |d d d d d f         z
  }|                    ddd                                          }|d d d d dfxx         | d         dz
  z  cc<   |d d d d dfxx         | d         dz
  z  cc<   |d d d d dfxx         d| d         z  dz
  z  cc<   t          j        |dz   fdz  |j                  }|	                    d          |dd dd f<   |dz
  |ddd f<   |dz
  |dd df<   |dz
  |d<   |S )N   r   r      )sizedtype)r   r   )
torchstackr   arangeflattenpermute
contiguouszerosr"   sum)r   num_relative_distancewindow_areacoordscoords_flattenrelative_coordsrelative_position_indexs          L/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/models/beit.pygen_relative_position_indexr3   =   s
   Q/!3KN8JQ8NORSS a.;q>1K[[^ < <el;WX>>Z>Z[[\\F]61--N$QQQ4Z0>!!!T111*3MMO%--aA66AACCOAAAqqq!GA 22AAAqqq!GA 22AAAqqq!GKN 2Q 66#ka/AA/E_Mbccc&5&9&9"&=&=ABBF#%:Q%>AqrrE"%:Q%>ABBE"$9A$=D!""    c                        e Zd ZU ej        j        e         ed<   	 	 	 	 	 	 	 ddededed	ed
e	de	de
eeef                  de
e         f fdZd Zdde
ej                 fdZ xZS )	Attention
fused_attn   F        Ndim	num_headsqkv_biasqkv_bias_separate	attn_drop	proj_dropr   attn_head_dimc	                    t                                                       || _        ||z  }	||}	|	| j        z  }
|	dz  | _        t	                      | _        || _        t          j        ||
dz  d          | _	        |rt          j
        t          j        |
                    | _        |                     dt          j        |
          d           t          j
        t          j        |
                    | _        nd | _        d | _        d | _        |r|| _        d|d         z  d	z
  d|d	         z  d	z
  z  dz   | _        t          j
        t          j        | j        |                    | _        |                     d
t)          |          d           nd | _        d | _        d | _        t          j        |          | _        t          j        |
|          | _        t          j        |          | _        d S )Ng      r    F)biask_bias)
persistentr   r   r   r1   )super__init__r;   scaler   r7   r=   nnLinearqkv	Parameterr$   r*   q_biasregister_bufferv_biasrC   r   r,   relative_position_bias_tabler3   r1   Dropoutr>   projr?   )selfr:   r;   r<   r=   r>   r?   r   r@   head_dimall_head_dim	__class__s              r2   rF   zAttention.__init__T   s    	")#$$H$.0%
(**!29S,"2??? 	,u{<'@'@AADK  5;|+D+DQV WWW,u{<'@'@AADKKDKDKDK 		0*D*+k!n*<q*@QUVEWZ[E[)\_`)`D&02D6	BB1D 1DD-  !:<WXc<d<dqv wwww#D04D-+/D(I..IlC00	I..r4   c                 b   | j         | j                            d                                       | j        d         | j        d         z  dz   | j        d         | j        d         z  dz   d          }|                    ddd                                          }|                    d          S )Nr#   r   r   r   )rO   r1   viewr   r(   r)   	unsqueezerR   relative_position_biass     r2   _get_rel_pos_biaszAttention._get_rel_pos_bias   s    !%!B(--b11"3374Q$"21"559Q$"21"55924? 4? 	 "8!?!?1a!H!H!S!S!U!U%//222r4   shared_rel_pos_biasc                    |j         \  }}}| j        |                     |          }nit          j        | j        | j        | j        f          }| j        r|                     |          }||z  }n!t          j	        || j        j
        |          }|                    ||d| j        d                              ddddd          }|                    d          \  }}	}
| j        rVd }| j        |                                 }|||z   }n||}t          j        ||	|
|| j        r| j        j        nd	          }nx|| j        z  }||	                    d
d          z  }| j        ||                                 z   }|||z   }|                    d          }|                     |          }||
z  }|                    dd                              |||          }|                     |          }|                     |          }|S )N)weightrB   r    r#   r   r   r      r9   )	attn_mask	dropout_pr:   )shaperL   rJ   r$   catrC   rN   r=   Flinearr^   reshaper;   r(   unbindr7   rO   r[   scaled_dot_product_attentiontrainingr>   prG   	transposesoftmaxrQ   r?   )rR   xr\   BNCrJ   r<   qkvrel_pos_biasattns                r2   forwardzAttention.forward   s   '1a;((1++CCy$+t{DK!HIIH% IhhqkkxhqxHHHkk!Q4>266>>q!Q1MM**Q--1a? 	L0<#5577&2#/2E#EL$02.1a&.2mC$.**  AA DJAB+++D0<d44666".11<<B<''D>>$''DqAKK1%%aA..IIaLLNN1r4   )r8   FFr9   r9   NNN)__name__
__module____qualname__r$   jitFinalbool__annotations__intfloatr   r   rF   r[   Tensorrx   __classcell__rU   s   @r2   r6   r6   Q   s        	%%%%
 "&+!!59+/,/ ,/,/ ,/ 	,/
  $,/ ,/ ,/ "%S/2,/ $C=,/ ,/ ,/ ,/ ,/ ,/\3 3 3- -hu|.D - - - - - - - -r4   r6   c                        e Zd Zddddddddej        eddfdedededed	ed
edededede	e         de
de
de	eeef                  de	e         f fdZdde	ej                 fdZ xZS )BlockF      @r9   Nr:   r;   r<   	mlp_ratio	scale_mlp
swiglu_mlpr?   r>   	drop_pathinit_values	act_layer
norm_layerr   r@   c           	         t                                                        ||          | _        t          |||||||          | _        |	dk    rt          |	          nt          j                    | _         ||          | _	        |r-t          |t          ||z            |r|nd |          | _        n-t          |t          ||z            ||r|nd |          | _        |	dk    rt          |	          nt          j                    | _        |
r^t          j        |
t!          j        |          z            | _        t          j        |
t!          j        |          z            | _        d S d\  | _        | _        d S )N)r;   r<   r>   r?   r   r@   r9   )in_featureshidden_featuresr   drop)r   r   r   r   r   )NN)rE   rF   norm1r6   rw   r   rH   Identity
drop_path1norm2r   r   mlpr   
drop_path2rK   r$   onesgamma_1gamma_2)rR   r:   r;   r<   r   r   r   r?   r>   r   r   r   r   r   r@   rU   s                  r2   rF   zBlock.__init__   s   " 	Z__
#'
 
 
	 2;R(9---R[]]Z__
 	 #C)O 4 4)2<::	  DHH  #C)O 4 4#)2<::  DH 2;R(9---R[]] 	4<ejoo(EFFDL<ejoo(EFFDLLL)3&DL$,,,r4   r\   c           	      .   | j         ||                     |                     |                     |          |                    z   }||                     |                     |                     |                              z   }n||                     | j         |                     |                     |          |          z            z   }||                     | j        |                     |                     |                    z            z   }|S )Nr\   )r   r   rw   r   r   r   r   r   )rR   ro   r\   s      r2   rx   zBlock.forward   s    <DOODIIdjjmmQdI$e$efffADOODHHTZZ]]$;$;<<<AADOODL499TZZ]]`s93t3t$tuuuADOODL488DJJqMM3J3J$JKKKAr4   ry   )rz   r{   r|   rH   GELUr   r   r   r   r   r   r   rF   r$   r   rx   r   r   s   @r2   r   r      s1        #!#$!!!+/"$'#,59+/54 5454 54 	54
 54 54 54 54 54 54 "%54  54 !54 "%S/254 $C=54 54 54 54 54 54n hu|.D        r4   r   c                   $     e Zd Z fdZd Z xZS )RelativePositionBiasc                 `   t                                                       || _        |d         |d         z  | _        d|d         z  dz
  d|d         z  dz
  z  dz   }t	          j        t          j        ||                    | _        | 	                    dt          |                     d S )Nr   r   r   r    r1   )rE   rF   r   r-   rH   rK   r$   r*   rO   rM   r3   )rR   r   r;   r,   rU   s       r2   rF   zRelativePositionBias.__init__   s    &&q>KN:!"[^!3a!7AA<NQR<R SVW W,.LEZ\e9f9f,g,g)68ST_8`8`aaaaar4   c                     | j         | j                            d                                       | j        dz   | j        dz   d          }|                    ddd                                          S )Nr#   r   r   r   )rO   r1   rW   r-   r(   r)   rY   s     r2   rx   zRelativePositionBias.forward  sm    !%!B4C_CdCdegChCh!i!n!nq $"2Q"6"< "<%--aA66AACCCr4   )rz   r{   r|   rF   rx   r   r   s   @r2   r   r      sS        b b b b bD D D D D D Dr4   r   c            /           e Zd ZdZddddddddd	d
dddddddedd	dddfdeeeeef         f         deeeeef         f         dedededededede	de
de	de	de
de
de
de
de
d ed!ee
         d"e	d#e	d$e	d%e
f. fd&Zd' Zd( Zej        j        d)             Zej        j        d@d*            Zej        j        dAd+            Zej        j        d,ej        fd-            ZdBdedee         fd.Z	 	 	 	 	 	 dCd0ej        d1eeeee         f                  d2e	d3e	d4e	d5ed6e	d,eeej                 eej        eej                 f         f         fd7Z	 	 	 dDd1eeee         f         d9e	d:e	fd;Zd< ZdAd=e	fd>Zd? Z  xZ!S )Er   zI Vision Transformer with support for patch or hybrid CNN input stage
          r      avg      Tr   Fr9   NgMbP?img_size
patch_sizein_chansnum_classesglobal_pool	embed_dimdepthr;   r<   r   r   r   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   r   use_abs_pos_embuse_rel_pos_biasuse_shared_rel_pos_biashead_init_scalec                    	
 t                                                       | _        | _        x _        x _         _        d _        d _        t          |||           _
         j
        j        }t           j
        d          r j
                                        n|t          j        t!          j        dd                     _        |r+t          j        t!          j        d|dz                       nd  _        t          j        |           _        |r!t-           j
        j                   _        nd  _        d t!          j        d||          D             t          j        
	 fd	t7          |          D                        _        fd
t7          |          D              _         j        dk    }|rt          j                    n
            _        |r           nt          j                     _         t          j        |           _!        |dk    rt          j"        |          nt          j                     _#         $                     j%                    j        tM           j        d           tM           j        d            '                                 tQ           j#        t          j"                  retM           j#        j)        d            j#        j)        j*        +                    |            j#        j,        j*        +                    |           d S d S )Nr   F)r   r   r   r   
feat_ratio)rl   )r   r;   c                 6    g | ]}|                                 S  )item).0ro   s     r2   
<listcomp>z!Beit.__init__.<locals>.<listcomp>I  s     JJJAqvvxxJJJr4   r   c                 n    g | ]1}t          
	|         rj        j        nd           2S )N)r:   r;   r<   r   r   r   r?   r>   r   r   r   r   )r   patch_embed	grid_size)r   ir   dprr   r   r   r   r;   r   r<   r   rR   r   r   s     r2   r   z!Beit.__init__.<locals>.<listcomp>J  ss     %# %# %#  #!##%((a&%':JTD,66PT  %# %# %#r4   c                 :    g | ]}t          d |           S )zblocks.)modulenum_chs	reductiondict)r   r   r   rs     r2   r   z!Beit.__init__.<locals>.<listcomp>Z  sA     ^ ^ ^KLD!yAFFF^ ^ ^r4   r   {Gz?std)-rE   rF   r   r   num_featureshead_hidden_sizer   num_prefix_tokensgrad_checkpointingr   r   num_patcheshasattrr   rH   rK   r$   r*   	cls_token	pos_embedrP   pos_dropr   r   rv   linspace
ModuleListrangeblocksfeature_infor   normfc_norm	head_droprI   headapply_init_weightsr   fix_init_weight
isinstancer^   datamul_rB   )rR   r   r   r   r   r   r   r   r;   r<   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   use_fc_normr   r   rU   s   `     ` `````  `` `` `    @@r2   rF   zBeit.__init__  s   4 	&&ENNND1DN!""'%!	
 
 
 &2-4T5E|-T-TdD'')))Zdek!Q	&B&BCCUdnek![1_i&P&PQQQjn
]333" 	% 4 ,6#! ! !D
 !%DJJ>5!I!IJJJm %# %# %# %# %# %# %# %# %# %# %# %# %# %# %# %# 5\\%# %# %# $ $ ^ ^ ^ ^ ^PUV[P\P\^ ^ ^ &%/%0KBKMMMjj6K6K	0;Nzz),,,I..9DqBIi555bkmm	

4%&&&>%$.c2222dn#....di++ 	6$)*4444I!&&777IN$$_55555	6 	6r4   c                     d }t          | j                  D ]K\  }} ||j        j        j        j        |dz               ||j        j        j        j        |dz              Ld S )Nc                 Z    |                      t          j        d|z                       d S )Ng       @)div_mathsqrt)paramlayer_ids     r2   rescalez%Beit.fix_init_weight.<locals>.rescaleo  s(    JJtyx0011111r4   r   )	enumerater   rw   rQ   r^   r   r   fc2)rR   r   r   layers       r2   r   zBeit.fix_init_weightn  s}    	2 	2 	2  )55 	= 	=OHeGEJO*/A>>>GEIM(-x!|<<<<	= 	=r4   c                    t          |t          j                  rbt          |j        d           t          |t          j                  r.|j        )t          j                            |j        d           d S d S d S t          |t          j                  rLt          j                            |j        d           t          j                            |j        d           d S d S )Nr   r   r         ?)	r   rH   rI   r   r^   rB   init	constant_r   )rR   ms     r2   r   zBeit._init_weightsv  s    a## 	-!(,,,,!RY'' -AF,>!!!&!,,,,,- -,>,>2<(( 	-Gafa(((Gah,,,,,	- 	-r4   c                 t    ddh}|                                  D ]\  }}d|v r|                    |           |S )Nr   r   rO   )named_parametersadd)rR   nwdn_s       r2   no_weight_decayzBeit.no_weight_decay  sK    K())++ 	 	DAq-22



r4   c                     || _         d S ry   )r   )rR   enables     r2   set_grad_checkpointingzBeit.set_grad_checkpointing  s    "(r4   c                 ,    t          dddg          }|S )Nz-^cls_token|pos_embed|patch_embed|rel_pos_bias)z^blocks\.(\d+)N)z^norm)i )stemr   r   )rR   coarsematchers      r2   group_matcherzBeit.group_matcher  s)    A-/CD
 
 
 r4   r   c                     | j         S ry   )r   )rR   s    r2   get_classifierzBeit.get_classifier  s
    yr4   c                     || _         ||| _        |dk    rt          j        | j        |          nt          j                    | _        d S Nr   )r   r   rH   rI   r   r   r   )rR   r   r   s      r2   reset_classifierzBeit.reset_classifier  sH    &"*D>IAooBIdnk:::SUS^S`S`			r4   NCHWro   indicesreturn_prefix_tokensr   
stop_early
output_fmtintermediates_onlyc                 6    |dv s
J d            |dk    }g }	t          t           j                  |          \  }
}|j        \  }}}                     |          }t          j         j                            |j        d         dd          |fd          } j	        
| j	        z   } 
                    |          } j                                         nd}t
          j                                        s|s j        }n j        d|dz            }t          |          D ]B\  }} |||	          }||
v r,|	                    |r                     |          n|           C j        r fd
|	D             } fd|	D             }	|r/ j                            ||f          \  fd|	D             }	t
          j                                        s|rt'          t)          |	|                    }	|r|	S                      |          }||	fS )a<   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if an int, if is a sequence, select by matching indices
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )r  NLCz)Output format must be one of NCHW or NLC.r  r   r#   r   rc   Nr   c                 6    g | ]}|d d dj         f         S r  r   r   yrR   s     r2   r   z.Beit.forward_intermediates.<locals>.<listcomp>  s/    SSSQqqq!D$:"::;SSSr4   c                 6    g | ]}|d d j         d f         S ry   r  r  s     r2   r   z.Beit.forward_intermediates.<locals>.<listcomp>  s0    RRRqQqqq$"8"9"99:RRRr4   c                     g | ]B}|                     d                               dddd                                          CS )r#   r   r    r   r   )rh   r(   r)   )r   r  rp   HWs     r2   r   z.Beit.forward_intermediates.<locals>.<listcomp>  sL    lllYZQYYq!Q33;;Aq!QGGRRTTlllr4   )r   lenr   rd   r   r$   re   r   expandr   r   rv   r}   is_scriptingr   appendr   r   dynamic_feat_sizelistzip)rR   ro   r  r  r   r  r  r  rh   intermediatestake_indices	max_indexr   heightwidthrv   r   r   blkprefix_tokensrp   r  r  s   `                   @@@r2   forward_intermediateszBeit.forward_intermediates  so   . _,,,.Y,,,&"6s4;7G7G"Q"Qi  g1feQIt~,,QWQZR@@!D!LLL>%DN"AMM!.2.?.Kt((***QU9!!## 	1: 	1[FF[)a-0F'' 	B 	BFAsA<888AL  $$T%@TYYq\\\qAAA ! 	SSSSS]SSSMRRRRMRRRM 	m#55vuoFFDAqllllll^klllMy%%'' 	D,@ 	D ]M!B!BCCM 	!  IIaLL-r4   r   
prune_norm
prune_headc                    t          t          | j                  |          \  }}| j        d|dz            | _        |rt          j                    | _        |r.t          j                    | _        |                     dd           |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r  r   rH   r   r   r   r  )rR   r  r-  r.  r&  r'  s         r2   prune_intermediate_layerszBeit.prune_intermediate_layers  s~     #7s4;7G7G"Q"Qik.9q=.1 	&DI 	);==DL!!!R(((r4   c                    |                      |          }t          j        | j                            |j        d         dd          |fd          }| j        
|| j        z   }|                     |          }| j        |                                 nd }| j	        D ]G}| j
        r1t          j                                        st          |||          }: |||          }H|                     |          }|S )Nr   r#   r   rc   r   )r   r$   re   r   r  rd   r   r   rv   r   r   r}   r   r   r   )rR   ro   rv   r*  s       r2   forward_featureszBeit.forward_features  s    QIt~,,QWQZR@@!D!LLL>%DN"AMM!.2.?.Kt((***QU; 	= 	=C& =uy/E/E/G/G =sA<HHHC|<<<IIaLLr4   
pre_logitsc                    | j         r>| j         dk    r'|d d | j        d f                             d          n|d d df         }|                     |          }|                     |          }|r|n|                     |          S )Nr   r   rc   r   )r   r   meanr   r   r   )rR   ro   r4  s      r2   forward_headzBeit.forward_head  s     	d=A=MQV=V=V!!!T+,,,-22q2999\]^_^_^_ab^b\cALLOONN10qqDIIaLL0r4   c                 Z    |                      |          }|                     |          }|S ry   )r3  r7  )rR   ro   s     r2   rx   zBeit.forward  s-    !!!$$a  r4   )TFry   )NFFFr  F)r   FT)"rz   r{   r|   __doc__r   r   r   r   strr   r   r   r   rF   r   r   r$   r}   ignorer   r  r  rH   Moduler
  r  r   r   r,  r1  r3  r7  rx   r   r   s   @r2   r   r     s        
 5868#$ !!$#!#%$&$&$&#,+/$(%*,1%*1Z6 Z6CsCx01Z6 c5c?23Z6 	Z6
 Z6 Z6 Z6 Z6 Z6 Z6 Z6 Z6 Z6 Z6 !Z6  "!Z6" "#Z6$ "%Z6& !'Z6( "%)Z6* "+Z6, #-Z6. &*/Z60 #1Z6 Z6 Z6 Z6 Z6 Z6x= = =- - - Y   Y) ) ) ) Y    Y	    a aC ahsm a a a a 8<).$$',A  A |A  eCcN34A  #'	A 
 A  A  A  !%A  
tEL!5tEL7I)I#JJ	KA  A  A  A J ./$#	 3S	>*  	   "   1 1$ 1 1 1 1      r4   r0  c                 "    | ddd dddddddd	|S )
Nr   )r    r   r   g?bicubicT)      ?r@  r@  zpatch_embed.projr   )urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer6  r   
first_conv
classifierr   )rA  kwargss     r2   _cfgrJ    s6    =t(   r4   ztimm/)	hf_hub_id)r      rL  r   )rK  rB  rD  iQU  )rK  r   )r       rM  )rK  r6  r   )rK  r   r6  r   gffffff?)rK  rD  r6  r   )z)beit_base_patch16_224.in22k_ft_in22k_in1kz)beit_base_patch16_384.in22k_ft_in22k_in1kz$beit_base_patch16_224.in22k_ft_in22kz*beit_large_patch16_224.in22k_ft_in22k_in1kz*beit_large_patch16_384.in22k_ft_in22k_in1kz*beit_large_patch16_512.in22k_ft_in22k_in1kz%beit_large_patch16_224.in22k_ft_in22kz*beitv2_base_patch16_224.in1k_ft_in22k_in1kz$beitv2_base_patch16_224.in1k_ft_in1kz%beitv2_base_patch16_224.in1k_ft_in22kz+beitv2_large_patch16_224.in1k_ft_in22k_in1kz%beitv2_large_patch16_224.in1k_ft_in1kz&beitv2_large_patch16_224.in1k_ft_in22kr?  Tc           	         |                      d|           } |                      d|           } i }|                                 D ]5\  }}d|v rd|v rT|j        j        j        j        \  }}}	}
|j        d         |
k    s|j        d         |	k    rt          ||	|
f||d          }n|d	k    rC|j        d
         |j        j        d
         k    r"d
}t          ||j        j	        |||d          }n|
                    d          ro|                    |d d                   }|j        |j        j        k    s|j        d         |j        d
         k    r!t          ||j        |j        j                  }|||<   7|S )Nmodelr   r1   zpatch_embed.proj.weightr#   rb   T)rE  	antialiasverboser   r   )new_sizer   rE  rP  rQ  rO   ir   )new_window_sizenew_bias_shape)getitemsr   rQ   r^   rd   r   r   r   r   endswithget_submodulerO   r   r   )
state_dictrO  rE  rP  out_dictrt   ru   OIr  r  r   r   s                r2   checkpoint_filter_fnr]  [  s   44J*55J H  ""    1$))$))*/6<JAq!Qwr{a172;!#3#3(F"/'    +!'!*0Ea0H"H"H !&*4"3+#  AA ZZ677 	##AdsdG,,Aw!8>>>!-PQBRVWVcdeVfBfBf-$%M#$#A#G  
 Or4   Fc                     |                     dd          }t          t          | |ft          t	          |d          d|}|S )Nout_indicesr    getter)r_  feature_cls)pretrained_filter_fnfeature_cfg)popr   r   r]  r   )variant
pretrainedrI  r_  rO  s        r2   _create_beitrg    sY    **]A..K gz1[hGGG  	 E Lr4   c           
      f    t          dddddddd          }t          dd
| it          |fi |}|S )Nr   r   r   r_   FT皙?r   r   r   r;   r   r   r   r   beit_base_patch16_224rf  )rk  r   rg  rf  rI  
model_argsrO  s       r2   rk  rk    s\    B"#G G GJ ffZf4PZKeKe^dKeKeffELr4   c           
      f    t          dddddddd          }t          dd
| it          |fi |}|S )NrL  r   r   r   FTri  r   r   r   r   r;   r   r   r   beit_base_patch16_384rf  )rq  rl  rm  s       r2   rq  rq    s\    s"#G G GJ ffZf4PZKeKe^dKeKeffELr4   c           	      d    t          ddddddd          }t          d
d	| it          |fi |}|S )Nr         FTh㈵>r   r   r   r;   r   r   r   beit_large_patch16_224rf  )rw  rl  rm  s       r2   rw  rw    sZ    R2$H H HJ ggjgDQ[LfLf_eLfLfggELr4   c           
      f    t          dddddddd          }t          dd
| it          |fi |}|S )NrL  r   rs  rt  FTru  rp  beit_large_patch16_384rf  )ry  rl  rm  s       r2   ry  ry    \    t2$H H HJ ggjgDQ[LfLf_eLfLfggELr4   c           
      f    t          dddddddd          }t          dd
| it          |fi |}|S )NrM  r   rs  rt  FTru  rp  beit_large_patch16_512rf  )r|  rl  rm  s       r2   r|  r|    rz  r4   c           
      f    t          dddddddd          }t          dd
| it          |fi |}|S )Nr   r   r   r_   FTru  rj  beitv2_base_patch16_224rf  )r~  rl  rm  s       r2   r~  r~    s\    B"$H H HJ hhzhTR\MgMg`fMgMghhELr4   c           	      d    t          ddddddd          }t          d
d	| it          |fi |}|S )Nr   rs  rt  FTru  rv  beitv2_large_patch16_224rf  )r  rl  rm  s       r2   r  r    sZ    R2$H H HJ ii
idS]NhNhagNhNhiiELr4   )r0  )r?  Tr9  ):r:  r   typingr   r   r   r   r   r$   torch.nnrH   torch.nn.functional
functionalrf   torch.utils.checkpointr   	timm.datar	   r
   timm.layersr   r   r   r   r   r   r   r   r   r   r   _builderr   	_featuresr   	_registryr   r   __all__r   r   r3   r=  r6   r   r   r   rJ  default_cfgsr]  rg  rk  rq  rw  ry  r|  r~  r  r   r4   r2   <module>r     sf   P  9 9 9 9 9 9 9 9 9 9 9 9 9 9                 - - - - - - A A A A A A A A c c c c c c c c c c c c c c c c c c g g g g g g g g g g g g + * * * * * + + + + + + < < < < < < < <(#U38_ # # # # #(f f f f f	 f f fR@ @ @ @ @BI @ @ @FD D D D D29 D D D"} } } } }29 } } }@    %$152 2 2 26 32 2 2
 -1D- - -
 37$3 3 3 37$ 33 3 3
 37$ 33 3 3
 .2T. . . 37$"(<3 3 3
 -1D"(<- - -
 .2T 5;O. . .
 48417K4 4 4
 .2T17K. . .
 /3d 5;O/ / /u?& ?& ? ?D' ' ' 'T                 $      $      $      4      D      r4   