
    NgK              
          d Z ddlZddlZddlmZmZ ddlZddlmZ ddl	mc m
Z ddlmZ ddlmZ ddlmZ ddlmZ  eej                            d	d                    dk    Z	 	 d-deeef         deeeef                  dedej        fdZdeeef         deedf         fdZ	 	 d.dedefdZdeeef         deedf         fdZ G d dej                  Z 	 	 d/deeef         deeef         fd Z! G d! d"ej                  Z"	 d0d#ed$ee         fd%Z#d&ed'ed(ej        d)ej        dej        f
d*Z$ G d+ d,ej                  Z%dS )1zf Relative position embedding modules and functions

Hacked together by / Copyright 2022 Ross Wightman
    N)OptionalTuple   )ndgrid)RegularGridInterpolator)Mlp)trunc_normal_TIMM_USE_SCIPY_INTERPFq_sizek_sizeclass_tokenreturnc           	         |
J d            t          j        t          t          j        | d                   t          j        | d                                                           d          }|d d d d d f         |d d d d d f         z
  }|                    ddd          }|d d d d dfxx         | d         dz
  z  cc<   |d d d d dfxx         | d         dz
  z  cc<   |d d d d dfxx         d| d         z  dz
  z  cc<   d| d         z  dz
  d| d         z  dz
  z  }|                    d          }|r4t          j        |g d          }||ddd f<   |dz   |dd df<   |dz   |d<   |	                                S )Nz-Different q & k sizes not currently supportedr   r      )r   r   r   r   r   r   )
torchstackr   arangeflattenpermutesumFpad
contiguous)r   r   r   coordsrelative_coordsnum_relative_distancerelative_position_indexs          U/var/www/html/ai-engine/env/lib/python3.11/site-packages/timm/layers/pos_embed_rel.pygen_relative_position_indexr!      s    >>J>>>[VAY 7 7fQi9P9PQQRRZZ[\]]FQQQ4Z(6!!!T111*+==O%--aA66OAAAqqq!Gq	A-AAAqqq!Gq	A-AAAqqq!GF1I 11]Q.1vay=13DE0 .11"55 B #$%(?"N"N)>122&)>)BA&(=(A%"--///    new_window_sizenew_bias_shape.c           	         |d         dz  dz
  |d         dz  dz
  f}| j         dk    r|\  }}}| j        \  }}}	||d         k    r||d         k    sJ ||k    s|	|k    rNt          j        j                            |                     d          |dd                              d          } n4| j         dk    sJ |\  }
}| j        \  }}|
|d         |d         z  z
  }t          ||z
  dz            }||f}|d         |d         k    s|d         |d         k    r|r| | d d d f         }| d | d d f         } nd }t          j        j                            | 	                    dd          
                    dd	|d         |d         f          |dd                              d	|
|z
            	                    dd          } |t          j        | |fd
          } | S )Nr   r   r      bicubicF)sizemodealign_corners      ?r   dim)ndimshaper   nn
functionalinterpolate	unsqueezesqueezeint	transposereshapeviewcat)rel_pos_biasr#   r$   dst_size_dst_hdst_wnum_attn_headssrc_hsrc_wdst_num_possrc_num_posnum_extra_tokenssrc_sizeextra_tokenss                  r     resize_rel_pos_bias_table_simplerG   I   sN   
  "Q&*OA,>,BQ,FGHA(5%'3'9$u##!(<(<(<(<E>>Ue^^ 8.::&&q))#	 ;  
 gajj   A%%%%'Q&2&8#^&(1+*CD&663>??h'A;(1+%%!)C)C $+-=,=,>,>,AB+,>.>->,>,AB# 8.::&&q!,,44aXa[(ST+5VWW#	 ;  
 d2{%5566yyA  '$y,)E1MMMr"   r'   Tinterpolation	antialiasc                    |                                  \  }}|\  }}||k    sJ ||k    r| j        }|                                 } t          |dz            }	t          |dz            }
t	          j        |                     dd                              d||	|	          |
|
f||          }|                    ||                              dd          }|                    |           |S | S )z
    Resample relative position bias table suggested in LeVit
    Adapted from: https://github.com/microsoft/Cream/blob/main/TinyViT/utils.py
    r+   r   r   )r(   r)   rI   )	r(   dtypefloatr5   r   r2   r   r8   to)position_bias_tablenew_sizerH   rI   L1nH1L2nH2
orig_dtypeS1S2$relative_position_bias_table_resizeds               r    resize_rel_pos_bias_table_levitrX   x   s    "&&((GBGB#::::	Rxx(.
17799s^^s^^/0}''1--221c2rBBb	0! 0! 0!, 155c2>>FFq!LL 	-,//
;;;33""r"   c                    t           rddlm} |d         dz  dz
  |d         dz  dz
  f}| j        dk    r4d}|\  }}}||d         k    r||d         k    sJ | j        \  }	}
}|
|f}d}nK| j        dk    sJ |\  }}| j        \  }}	||d         |d         z  z
  }t          ||z
  dz            }||f}d}|d         |d         k    s|d         |d         k    r|r| | d	d	d	f         }| d	| d	d	f         } nd	}d
 fd} ||d         |d                   } ||d         |d                   }t          j        |          t          j        |          g}|d         dz  }|d         dz  }t          j        | |dz   d          }t          j        | |dz   d          }t          ||          }g }t          |	          D ]R}|r?| d	d	|f                             |d         |d                                                   }n!| |d	d	d	d	f                                         }t           rt|                    |||                                d          }t          j         |||                                                                        | j                  }nEt'          ||          } ||                                                              | j                  }|r|                    dd          }|                    |           T|rt          j        |d          } nt          j        |d          } ||sJ t          j        | |fd          } | S )a   Resize relative position bias table using more advanced interpolation.

    Modified from code in Microsoft Unilm (https://github.com/microsoft/unilm) repo (BeiT, BeiT-v2, etc).

    https://github.com/microsoft/unilm/blob/5255d52de86dad642810f5849dd357769346c1d7/beit/run_class_finetuning.py#L351

    Args:
        rel_pos_bias:
        new_window_size:
        new_bias_shape:

    Returns:

    r   )r2   r   r   r&   Fr+   TNc                 $    | d||z  z
  z  d|z
  z  S )N      ? )arns      r    geometric_progressionz8resize_rel_pos_bias_table.<locals>.geometric_progression   s    a1f%q11r"   c                 4   d\  }}||z
  dk    r/||z   dz  } 
d|| dz            }||dz  k    r|}n|}||z
  dk    /g }d}t          | dz            D ]"}|                    |           |||dz   z  z  }#d t          |          D             }	|	dgz   |z   S )N)g)\(?g      ?gư>       @r   r   c                     g | ]}| S r\   r\   ).0r<   s     r    
<listcomp>z<resize_rel_pos_bias_table.<locals>._calc.<locals>.<listcomp>   s    ///AaR///r"   r   )rangeappendreversed)srcdstleftrightqgpdiscurir_idsr`   s             r    _calcz(resize_rel_pos_bias_table.<locals>._calc   s    #KD%$,%%E\S(**1a::q==EED $,%% CC3!8__ $ $

3qQU|#//#///EA3;$$r"   rb   g?r[   cubic)kindr   r,   )
_USE_SCIPYscipyr2   r.   r/   r5   r   tensorr   r   rf   r8   rL   interp2dnumpyTensorr   rM   devicer   rg   r9   ) r:   r#   r$   r2   r;   rD   r<   r=   r>   r?   r@   rA   rE   has_flat_shaperB   rC   rF   rs   yxyxtytxdydxdyxall_rel_pos_biasrq   zfr^   r`   s                                   @r    resize_rel_pos_bias_tabler      s
   &  &%%%%%%"Q&*OA,>,BQ,FGHA(5%##!(<(<(<(<'3'9$u5> A%%%%'Q&2&8#^&(1+*CD&663>??h'{hqk!!Xa[HQK%?%? 	 ')9(9(:(:AAA(=>L'(:*:):(:AAA(=>LLL	2 	2 	2	% 	% 	% 	% 	%$ E(1+x{++E(1+x{++l1oou|A/ a[Ca[C\2#rCx--\2#rCx--Rnn ~&& 	' 	'A 2 A&++HQK!EEKKMM AAAqqq)//11 	@((Aqwwyyw(GGL2r++6688;;L<OPP
 ,B22AcFF%%''**<+>?? "FF2qMM##A&&&& 	> 9%52>>>LL 9%51===L#!!!! 9lL%AqIIILr"   c                   d     e Zd ZdZd
 fd	Zd Zdej        fdZdde	ej                 fd	Z
 xZS )
RelPosBiasz_ Relative Position Bias
    Adapted from Swin-V1 relative position bias impl, modularized.
    r   c                    t                                                       |dk    sJ || _        |d         |d         z  | _        | j        |z   fdz  |fz   | _        d|d         z  dz
  d|d         z  dz
  z  d|z  z   }t          j        t          j        ||                    | _	        | 
                    dt          | j        |dk                                  d          d	           |                                  d S )
Nr   r   r   r&   r   )r   r   F
persistent)super__init__window_sizewindow_area
bias_shaper0   	Parameterr   zerosrelative_position_bias_tableregister_bufferr!   r8   init_weights)selfr   	num_headsprefix_tokensr   	__class__s        r    r   zRelPosBias.__init__  s   !!!!&&q>KN:+m;=AYLP!"[^!3a!7AA<NQR<R SVWZgVg g,.LEZ\e9f9f,g,g)%'(8mVWFWXXX]]^`aa 	 	
 	
 	
 	r"   c                 2    t          | j        d           d S Ng{Gz?)std)r	   r   r   s    r    r   zRelPosBias.init_weights"  s    d7SAAAAAAr"   r   c                     | j         | j                 }|                    | j                                      ddd          }|                    d                                          S )Nr   r   r   )r   r   r8   r   r   r3   r   r   relative_position_biass     r    get_biaszRelPosBias.get_bias%  s^    !%!B4C_!`!7!<!<T_!M!M!U!UVWYZ\]!^!^%//22==???r"   Nshared_rel_posc                 0    ||                                  z   S Nr   r   attnr   s      r    forwardzRelPosBias.forward+      dmmoo%%r"   r   r   __name__
__module____qualname____doc__r   r   r   r{   r   r   r   __classcell__r   s   @r    r   r     s              "B B B@%, @ @ @ @& &HU\,B & & & & & & & &r"   r   r   swinwin_sizepretrained_win_sizec                    |dv sJ t          j        | d         dz
   | d                                       t           j                  }t          j        | d         dz
   | d                                       t           j                  }t          j        t          ||                    }|                    ddd                                          }|dk    r|d         dk    rA|d d d d dfxx         |d         dz
  z  cc<   |d d d d dfxx         |d         dz
  z  cc<   n@|d d d d dfxx         | d         dz
  z  cc<   |d d d d dfxx         | d         dz
  z  cc<   |dz  }t          j        |          t          j	        d|
                                z             z  t          j	        d          z  }n>t          j        |          t          j        d|
                                z             z  }|S )N)r   crr   r   r   r      r[   )r   r   rM   float32r   r   r   r   signlog2absmathlog)r   r   r)   relative_coords_hrelative_coords_wrelative_coords_tables         r    gen_relative_log_coordsr   /  s?   
 >!!!!x{Q%7!EEHHWWx{Q%7!EEHHWW!K/@BS(T(TUU199!QBBMMOOv~~q!A%%!!!!QQQ'***/B1/E/IJ***!!!!QQQ'***/B1/E/IJ****!!!!QQQ'***x{Q?***!!!!QQQ'***x{Q?***" %
+@ A AEJ'++---E/ E/ !/151!> !&
+@ A AEI'++---E/ E/ !/ ! r"   c                   h     e Zd ZdZ	 	 	 	 	 d fd	Zdej        fd	Zddeej                 fdZ	 xZ
S )	RelPosMlpz Log-Coordinate Relative Position MLP
    Based on ideas presented in Swin-V2 paper (https://arxiv.org/abs/2111.09883)

    This impl covers the 'swin' implementation as well as two timm specific modes ('cr', and 'rw')
    r      r   r   r   c                 t   t                                                       || _        | j        d         | j        d         z  | _        || _        || _        | j        fdz  |fz   | _        |dk    r"t          j                    | _	        d| _
        d}n!t          j                    | _	        d | _
        d}t          d||t          j        |d	          | _        |                     d
t!          |                              d          d           |                     dt%          |||          d           d S )Nr   r   r   r      )TFT)g      ?g        )hidden_featuresout_features	act_layerbiasdropr   r   Fr   rel_coords_log)r)   )r   r   r   r   r   r   r   r0   Sigmoidbias_act	bias_gainIdentityr   ReLUmlpr   r!   r8   r   )	r   r   r   
hidden_dimr   r)   pretrained_window_sizemlp_biasr   s	           r    r   zRelPosMlp.__init__R  sO    	&+A.1A!1DD*"+-1YL@6>>JLLDMDN$HHKMMDM!DNH&"g
 
 
 	%'4499"== 	 	 	 	 	#K1GdSSS 	 	 	 	 	 	r"   r   c                    |                      | j                  }| j        @|                    d| j                  | j                 }|                    | j                  }|                    ddd          }|                     |          }| j        
| j        |z  }| j	        r#t          j        || j	        d| j	        dg          }|                    d                                          S )Nr   r   r   r   )r   r   r   r8   r   r   r   r   r   r   r   r   r3   r   r   s     r    r   zRelPosMlp.get_bias~  s    !%$*=!>!>'3%;%@%@T^%T%TUYUq%r"%;%@%@%Q%Q"!7!?!?1a!H!H!%/E!F!F>%%)^6L%L" 	s%&U+ADDVXY[_[mopCq%r%r"%//22==???r"   Nr   c                 0    ||                                  z   S r   r   r   s      r    r   zRelPosMlp.forward  r   r"   )r   r   r   r   r   r   )r   r   r   r   r   r   r{   r   r   r   r   r   s   @r    r   r   L  s          #)* * * * * *X@%, @ @ @ @& &HU\,B & & & & & & & &r"   r   lengthmax_relative_positionc                     || dz
  }d|z  dz   }t          j        | | |          }t          |           D ];}t          |           D ])}||z
  |z   }t          ||z
            |k    r!d||||f<   *<|S )a  Generate a one_hot lookup tensor to reindex embeddings along one dimension.

    Args:
        length: the length to reindex to.
        max_relative_position: the maximum relative position to consider.
            Relative position embeddings for distances above this threshold
            are zeroed out.
    Returns:
        a lookup Tensor of size [length, length, vocab_size] that satisfies
            ret[n,m,v] = 1{m - n + max_relative_position = v}.
    Nr   r   )r   r   rf   r   )r   r   
vocab_sizeretrq   r   vs          r    generate_lookup_tensorr     s     $ &
**Q.J
+ffj
1
1C6]]  v 	 	AA--A1q5zz111C1aLL		
 Jr"   heightwidthheight_lookupwidth_lookupc                     t          j        d| |          }t          j        d||          }||z  }|                    | j        d         ||          S )a\  Reindex 2d relative position bias with 2 independent einsum lookups.

    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py

    Args:
        relative_position_tensor: tensor of shape
            [..., vocab_height, vocab_width, ...].
        height: height to reindex to.
        width: width to reindex to.
        height_lookup: one-hot height lookup
        width_lookup: one-hot width lookup
    Returns:
        reindexed_tensor: a Tensor of shape
            [..., height * width, height * width, ...]
    znhw,ixh->nixwznixw,jyw->nijxyr   )r   einsumr7   r/   )relative_position_tensorr   r   r   r   reindexed_tensorareas          r    reindex_2d_einsum_lookupr     sY    . |O5M}]]|$57GVVE>D##$<$B1$EtTRRRr"   c                   d     e Zd ZdZd
 fd	Zd Zdej        fdZdde	ej                 fd	Z
 xZS )RelPosBiasTfz Relative Position Bias Impl (Compatible with Tensorflow MaxViT models)
    Adapted from:
     https://github.com/google-research/maxvit/blob/2e06a7f1f70c76e64cd3dabe5cd1b8c1a23c9fb7/maxvit/models/attention_utils.py
    r   c                 *   t                                                       |dk    sJ || _        |d         |d         z  | _        || _        d|d         z  dz
  }d|d         z  dz
  }| j        ||f| _        t          j        t          j	        | j                            | _
        |                     dt          |d                   d           |                     dt          |d                   d           |                                  d S )Nr   r   r   r   Fr   r   )r   r   r   r   r   r   r0   r   r   r   r   r   r   r   )r   r   r   r   vocab_heightvocab_widthr   s         r    r   zRelPosBiasTf.__init__  s   !!!!&&q>KN:";q>)A-+a.(1,><E,.LT_9U9U,V,V)_.D[QR^.T.Tafggg^-CKPQN-S-S`efffr"   c                 R    t           j                            | j        d           d S r   )r0   initnormal_r   r   s    r    r   zRelPosBiasTf.init_weights  s#    
9sCCCCCr"   r   c                 r    t          | j        | j        d         | j        d         | j        | j                  S )Nr   r   )r   r   r   r   r   r   s    r    r   zRelPosBiasTf.get_bias  s<    '-QQ
 
 	
r"   Nr   c                 0    ||                                  z   S r   r   r   s      r    r   zRelPosBiasTf.forward  r   r"   r   r   r   r   s   @r    r   r     s              D D D
%, 
 
 
 
& &HU\,B & & & & & & & &r"   r   )NF)r'   T)r   r   r   )&r   r   ostypingr   r   r   torch.nnr0   torch.nn.functionalr1   r   gridr   r2   r   r   r   weight_initr	   r5   environgetrv   boolr{   r!   rG   strrX   r   Moduler   r   r   r   r   r   r\   r"   r    <module>r     sC     				 " " " " " " " "                       0 0 0 0 0 0       & & & & & &S 7;;<<q@

 -1!10 10c3h10sCx)10 10 \	10 10 10 10h,sCx, c3h, , , ,d '	# # # 	# # # #@qsCxq c3hq q q qh &  &  &  &  &  &  &  &J 06! !S/!"38_! ! ! !:@& @& @& @& @&	 @& @& @&J 04 '}   :SS S |	S
 lS \S S S S:"& "& "& "& "&29 "& "& "& "& "&r"   