
    g                     ,   d Z ddlZddlmZ ddlmZmZmZmZ ddl	Z	ddl
Z	ddl	mZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZ ddlmZmZmZmZmZ ddl m!Z!  ej"        e#          Z$dZ%dZ&dZ'dZ(e G d de                      Z) G d dej*                  Z+ G d dej*                  Z, G d dej*                  Z- G d dej*                  Z. G d dej*                  Z/ G d d ej*                  Z0 G d! d"ej*                  Z1 G d# d$ej*                  Z2 G d% d&ej*                  Z3 G d' d(ej*                  Z4 G d) d*ej*                  Z5 G d+ d,ej*                  Z6 G d- d.ej*                  Z7 G d/ d0e          Z8 ed1e'           G d2 d3e8                      Z9 ed4e'           G d5 d6e8                      Z: ed7e'           G d8 d9e8                      Z; ed:e'           G d; d<e8                      Z<dS )=zPyTorch Bros model.    N)	dataclass)ListOptionalTupleUnion)nn)CrossEntropyLoss   )ACT2FN))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)ModelOutputadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )
BrosConfigzjinho8345/bros-base-uncasedr   aK  
    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`BrosConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`BrosProcessor`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)

        bbox ('torch.FloatTensor' of shape '(batch_size, num_boxes, 4)'):
            Bounding box coordinates for each token in the input sequence. Each bounding box is a list of four values
            (x1, y1, x2, y2), where (x1, y1) is the top left corner, and (x2, y2) is the bottom right corner of the
            bounding box.

        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

        bbox_first_token_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to indicate the first token of each bounding box. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)

        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.

        return_dict (`bool`, *optional*):
            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
c                       e Zd ZU dZdZeej                 ed<   dZ	ej        ed<   dZ
ej        ed<   dZeeej                          ed<   dZeeej                          ed<   dS )BrosSpadeOutputa  
    Base class for outputs of token classification models.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided) :
            Classification loss.
        initial_token_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
            Classification scores for entity initial tokens (before SoftMax).
        subsequent_token_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length+1)`):
            Classification scores for entity sequence tokens (before SoftMax).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nlossinitial_token_logitssubsequent_token_logitshidden_states
attentions)__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__r   r   r   r   r         b/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/bros/modeling_bros.pyr   r      s          . )-D(5$
%,,,.2%+22215U.5558<M8E%"345<<<59Ju01299999r)   r   c                   B     e Zd Z fdZdej        dej        fdZ xZS )BrosPositionalEmbedding1Dc                     t          t          |                                            |j        | _        ddt	          j        d| j        d          | j        z  z  z  }|                     d|           d S )Nr   i'          g       @inv_freq)superr,   __init__dim_bbox_sinusoid_emb_1dr%   arangeregister_buffer)selfconfigr/   	__class__s      r*   r1   z"BrosPositionalEmbedding1D.__init__   st    '..77999(.(G%el3(EsKKdNkkl
 	Z22222r)   pos_seqreturnc                 .   |                                 }|\  }}}|                    |||d          | j                            ddd| j        dz            z  }t	          j        |                                |                                gd          }|S )Nr      dim)sizeviewr/   r2   r%   catsincos)r5   r8   seq_sizeb1b2b3sinusoid_inppos_embs           r*   forwardz!BrosPositionalEmbedding1D.forward   s    <<>>
B||BB22T]5G5G1aQUQnrsQs5t5tt)\--//1A1A1C1CD"MMMr)   r!   r"   r#   r1   r%   TensorrJ   __classcell__r7   s   @r*   r,   r,      s^        3 3 3 3 3u|         r)   r,   c                   B     e Zd Z fdZdej        dej        fdZ xZS )BrosPositionalEmbedding2Dc                     t          t          |                                            |j        | _        t	          |          | _        t	          |          | _        d S N)r0   rP   r1   dim_bboxr,   	x_pos_emb	y_pos_embr5   r6   r7   s     r*   r1   z"BrosPositionalEmbedding2D.__init__   sK    '..7799926::26::r)   bboxr9   c                 8   g }t          | j                  D ]l}|dz  dk    r1|                    |                     |d|f                              <|                    |                     |d|f                              mt          j        |d          }|S )Nr;   r   .r<   r=   )rangerS   appendrT   rU   r%   rA   )r5   rW   stackibbox_pos_embs        r*   rJ   z!BrosPositionalEmbedding2D.forward   s    t}%% 	; 	;A1uzzT^^DaL99::::T^^DaL99::::yB///r)   rK   rN   s   @r*   rP   rP      s^        ; ; ; ; ;EL U\        r)   rP   c                   4     e Zd Z fdZdej        fdZ xZS )BrosBboxEmbeddingsc                     t          t          |                                            t          |          | _        t          j        |j        |j        d          | _	        d S )NF)bias)
r0   r_   r1   rP   bbox_sinusoid_embr   Lineardim_bbox_sinusoid_emb_2ddim_bbox_projectionbbox_projectionrV   s     r*   r1   zBrosBboxEmbeddings.__init__   sV     $''00222!:6!B!B!y)H&Jdkpqqqr)   rW   c                     |                     dd          }|d d d d d d d f         |d d d d d d d f         z
  }|                     |          }|                     |          }|S )Nr   r   )	transposerb   rf   )r5   rW   bbox_tbbox_posr]   s        r*   rJ   zBrosBboxEmbeddings.forward   s}    1%%$111aaa-(6!!!T111aaa-+@@--h77++L99r)   rK   rN   s   @r*   r_   r_      sZ        r r r r r
EL        r)   r_   c                        e Zd ZdZ fdZ	 	 	 	 	 ddeej                 deej                 deej                 deej                 d	ed
ej        fdZ	 xZ
S )BrosTextEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    t                                                       t          j        |j        |j        |j                  | _        t          j        |j        |j                  | _	        t          j        |j
        |j                  | _        t          j        |j        |j                  | _        t          j        |j                  | _        t#          |dd          | _        |                     dt)          j        |j                                      d                     |                     dt)          j        | j                                        t(          j        | j        j                  d	
           d S )N)padding_idxepsposition_embedding_typeabsoluteposition_ids)r   r<   token_type_idsdtypedeviceF)
persistent)r0   r1   r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutgetattrrq   r4   r%   r3   expandzerosrs   r?   longrw   rV   s     r*   r1   zBrosTextEmbeddings.__init__   sN   !|F,=v?Q_e_rsss#%<0NPVPb#c#c %'\&2H&J\%]%]" f&8f>STTTz&"<=='.v7PR\']']$^U\&:X-Y-Y-`-`ah-i-ijjjK!&&((j(/  
  	 	
 	
 	
 	
 	
r)   Nr   	input_idsrt   rs   inputs_embedspast_key_values_lengthr9   c                    ||                                 }n|                                 d d         }|d         }|| j        d d |||z   f         }|mt          | d          r2| j        d d d |f         }|                    |d         |          }	|	}n+t          j        |t
          j        | j        j                  }|| 	                    |          }| 
                    |          }
||
z   }| j        dk    r|                     |          }||z  }|                     |          }|                     |          }|S )Nr<   r   rt   r   ru   rr   )r?   rs   hasattrrt   r   r%   r   r   rw   r}   r   rq   r   r   r   )r5   r   rt   rs   r   r   input_shape
seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr   
embeddingsr   s                r*   rJ   zBrosTextEmbeddings.forward   sk     #..**KK',,..ss3K ^
,QQQ0FVlIl0l-lmL!t-.. m*.*=aaa*n*M'3J3Q3QR]^_R`bl3m3m0!A!&[
SWSdSk!l!l!l  00;;M $ : :> J J"%::
':55"&":":<"H"H--J^^J//
\\*--
r)   )NNNNr   )r!   r"   r#   r$   r1   r   r%   rL   intrJ   rM   rN   s   @r*   rl   rl      s        QQ
 
 
 
 
4 -115/304&'$ $EL)$ !.$ u|,	$
  -$ !$$ 
$ $ $ $ $ $ $ $r)   rl   c                   4    e Zd Z fdZdej        fdZ	 	 	 	 	 	 ddej        dej        deej                 d	eej                 d
eej                 deej                 deeeej	                                   deej                 deej                 fdZ
 xZS )BrosSelfAttentionc                 @   t                                                       |j        |j        z  dk    r0t	          |d          s t          d|j         d|j         d          |j        | _        t          |j        |j        z            | _        | j        | j        z  | _        t          j
        |j        | j                  | _        t          j
        |j        | j                  | _        t          j
        |j        | j                  | _        t          j        |j                  | _        t#          |dd          | _        | j        dk    s| j        d	k    r6|j        | _        t          j        d
|j        z  dz
  | j                  | _        |j        | _        d S )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()rq   rr   relative_keyrelative_key_queryr;   r   )r0   r1   r{   num_attention_headsr   
ValueErrorr   attention_head_sizeall_head_sizer   rc   querykeyvaluer   attention_probs_dropout_probr   r   rq   r~   ry   distance_embedding
is_decoderrV   s     r*   r1   zBrosSelfAttention.__init__  s    ::a??PVXhHiHi?8F$6 8 8 48 8 8  
 $*#= #&v'9F<V'V#W#W !58PPYv143EFF
9V/1CDDYv143EFF
z&"EFF'.v7PR\']']$'>99T=Y]q=q=q+1+ID(&(l1v7U3UXY3Y[_[s&t&tD# +r)   xc                     |                                 d d         | j        | j        fz   } |j        | }|                    dddd          S )Nr<   r   r;   r   r
   )r?   r   r   r@   permute)r5   r   new_x_shapes      r*   transpose_for_scoresz&BrosSelfAttention.transpose_for_scores3  sU    ffhhssm$$'
 
 AFK yyAq!$$$r)   NFr   r]   attention_mask	head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsr9   c	                 h   |                      |          }	|d u}
|
r||d         }|d         }|}n4|
rS|                     |                     |                    }|                     |                     |                    }|}n||                     |                     |                    }|                     |                     |                    }t	          j        |d         |gd          }t	          j        |d         |gd          }nP|                     |                     |                    }|                     |                     |                    }|                     |	          }| j        r||f}t	          j        ||                    dd                    }| j	        dk    s| j	        dk    r4|
                                d         }t	          j        |t          j        |j        	                              dd          }t	          j        |t          j        |j        	                              dd          }||z
  }|                     || j        z   dz
            }|                    |j        
          }| j	        dk    rt	          j        d||          }||z   }n?| j	        dk    r4t	          j        d||          }t	          j        d||          }||z   |z   }|j        \  }}}}|                    ||||          }|                    g d          }t	          j        d||f          }||z   }|t-          j        | j                  z  }|||z   } t3          j        d          |          }|                     |          }|||z  }t	          j        ||          }|                    dddd                                          }|
                                d d         | j        fz   } |j        | }|r||fn|f}| j        r||fz   }|S )Nr   r   r;   r=   r<   r   r   ru   rv   zbhld,lrd->bhlrzbhrd,lrd->bhlr)r;   r   r   r
   zbnid,bijd->bnijr
   )r   r   r   r   r%   rA   r   matmulrh   rq   r?   r3   r   rw   r@   r   r~   torv   einsumshaper   mathsqrtr   r   Softmaxr   
contiguousr   )r5   r   r]   r   r   r   r   r   r   mixed_query_layeris_cross_attention	key_layervalue_layerquery_layerattention_scoresr   position_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_key
batch_sizen_headd_headbbox_pos_scoresattention_probscontext_layernew_context_layer_shapeoutputss                                  r*   rJ   zBrosSelfAttention.forward;  sn    !JJ}55
 3$> 	O."<&q)I(+K3NN 	O11$((;P2Q2QRRI33DJJ?T4U4UVVK3NN'11$((=2I2IJJI33DJJ}4M4MNNK	>!#4i"@aHHHI)^A%6$D!LLLKK11$((=2I2IJJI33DJJ}4M4MNNK//0ABB? 	6 (5N !<Y5H5HR5P5PQQ'>99T=Y]q=q=q&++--a0J"\*EJ}OcdddiijlnoppN"\*EJ}OcdddiijkmoppN%6H#'#:#:8dFb;bef;f#g#g #7#:#:AR#:#S#S +~==+0<8H+Wk+l+l(#36N#N  -1EEE16>NP[]q1r1r./4|<LiYm/n/n,#36T#TWs#s  2=1B.
FJ#((ZVTT#++LLL99,'8;:UVV+o=+di8P.Q.QQ%/.@ -"*,,,-=>> ,,77  -	9O_kBB%--aAq99DDFF"/"4"4"6"6ss";t?Q>S"S**,CD6G]=/22mM]? 	2 11Gr)   NNNNNF)r!   r"   r#   r1   r%   rL   r   r   r   r&   rJ   rM   rN   s   @r*   r   r     s,       , , , , ,0%el % % % % 26,08<9=DH49f f|f lf !.	f
 EL)f  (5f !) 6f !uU->'?!@Af $EL1f 
u|	f f f f f f f fr)   r   c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )BrosSelfOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j        |j	                  | _
        d S Nro   )r0   r1   r   rc   r{   denser   r   r   r   r   rV   s     r*   r1   zBrosSelfOutput.__init__  sf    Yv163EFF
f&8f>STTTz&"<==r)   r   input_tensorr9   c                     |                      |          }|                     |          }|                     ||z             }|S rR   r   r   r   r5   r   r   s      r*   rJ   zBrosSelfOutput.forward  @    

=11]33}|'CDDr)   rK   rN   s   @r*   r   r     i        > > > > >U\  RWR^        r)   r   c                       e Zd Z fdZd Z	 	 	 	 	 	 ddej        dej        deej                 deej                 d	eej                 d
eej                 deeeej	                                   dee
         deej                 fdZ xZS )BrosAttentionc                     t                                                       t          |          | _        t	          |          | _        t                      | _        d S rR   )r0   r1   r   r5   r   outputsetpruned_headsrV   s     r*   r1   zBrosAttention.__init__  sI    %f--	$V,,EEr)   c                    t          |          dk    rd S t          || j        j        | j        j        | j                  \  }}t          | j        j        |          | j        _        t          | j        j        |          | j        _        t          | j        j	        |          | j        _	        t          | j
        j        |d          | j
        _        | j        j        t          |          z
  | j        _        | j        j        | j        j        z  | j        _        | j                            |          | _        d S )Nr   r   r=   )lenr   r5   r   r   r   r   r   r   r   r   r   r   union)r5   headsindexs      r*   prune_headszBrosAttention.prune_heads  s    u::??F7I)I)	
 
u -TY_eDD	*49=%@@	,TY_eDD	.t{/@%QOOO )-	(EE

(R	%"&)"?$)B_"_	 -33E::r)   NFr   r]   r   r   r   r   r   r   r9   c	           
          |                      ||||||||          }	|                     |	d         |          }
|
f|	dd          z   }|S )Nr   r]   r   r   r   r   r   r   r   r   )r5   r   )r5   r   r]   r   r   r   r   r   r   self_outputsattention_outputr   s               r*   rJ   zBrosAttention.forward  si     yy'%)"7#9)/ ! 	
 	
  ;;|AFF#%QRR(88r)   r   )r!   r"   r#   r1   r   r%   rL   r   r   r&   boolrJ   rM   rN   s   @r*   r   r     s	       " " " " "; ; ;2 26,08<9=DH,1 | l !.	
 EL)  (5 !) 6 !uU->'?!@A $D> 
u|	       r)   r   c                   B     e Zd Z fdZdej        dej        fdZ xZS )BrosIntermediatec                    t                                                       t          j        |j        |j                  | _        t          |j        t                    rt          |j                 | _        d S |j        | _        d S rR   )r0   r1   r   rc   r{   intermediate_sizer   
isinstance
hidden_actstrr   intermediate_act_fnrV   s     r*   r1   zBrosIntermediate.__init__  sn    Yv163KLL
f'-- 	9'-f.?'@D$$$'-'8D$$$r)   r   r9   c                 Z    |                      |          }|                     |          }|S rR   )r   r   )r5   r   s     r*   rJ   zBrosIntermediate.forward  s,    

=1100??r)   rK   rN   s   @r*   r   r     s^        9 9 9 9 9U\ el        r)   r   c                   P     e Zd Z fdZdej        dej        dej        fdZ xZS )
BrosOutputc                    t                                                       t          j        |j        |j                  | _        t          j        |j        |j                  | _        t          j	        |j
                  | _        d S r   )r0   r1   r   rc   r   r{   r   r   r   r   r   r   rV   s     r*   r1   zBrosOutput.__init__  sf    Yv79KLL
f&8f>STTTz&"<==r)   r   r   r9   c                     |                      |          }|                     |          }|                     ||z             }|S rR   r   r   s      r*   rJ   zBrosOutput.forward   r   r)   rK   rN   s   @r*   r   r     r   r)   r   c                       e Zd Z fdZ	 	 	 	 	 	 ddej        dej        deej                 deej                 deej                 d	eej                 d
eeeej                                   dee	         deej                 fdZ
d Z xZS )	BrosLayerc                 ~   t                                                       |j        | _        d| _        t	          |          | _        |j        | _        |j        | _        | j        r-| j        st          |  d          t	          |          | _	        t          |          | _        t          |          | _        d S )Nr   z> should be used as a decoder model if cross attention is added)r0   r1   chunk_size_feed_forwardseq_len_dimr   	attentionr   add_cross_attention	Exceptioncrossattentionr   intermediater   r   rV   s     r*   r1   zBrosLayer.__init__  s    '-'E$&v.. +#)#= # 	8? i4 g g ghhh"/"7"7D,V44 ((r)   NFr   r]   r   r   r   r   r   r   r9   c	           	      
   |
|d d         nd }	|                      ||||||	          }
|
d         }| j        r|
dd         }|
d         }n
|
dd          }d }| j        rp|nt          | d          rt          d|  d          |
|d	d          nd }|                     |||||||          }|d         }||dd         z   }|d         }||z   }t          | j        | j        | j        |          }|f|z   }| j        r||fz   }|S )
Nr;   )r]   r   r   r   r   r   r   r<   r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r   )	r  r   r   r  r  r   feed_forward_chunkr   r   )r5   r   r]   r   r   r   r   r   r   self_attn_past_key_valueself_attention_outputsr   r   present_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputslayer_outputs                     r*   rJ   zBrosLayer.forward  s    :H9S>"1"#5#5Y] !%%)/3 "0 "
 "
 2!4 ? 	1,QrT2G 6r :,QRR0G'+$? 	Q4@t-..  ed  e  e  e  
 @N?Yrss(;(;_c%&*&9&9 %&)!' '#  7q9 7" ==G ,C2+F( 14P P0#(	
 
  /G+ ? 	5!2 44Gr)   c                 \    |                      |          }|                     ||          }|S rR   )r  r   )r5   r   intermediate_outputr  s       r*   r  zBrosLayer.feed_forward_chunk[  s2    "//0@AA{{#68HIIr)   r   )r!   r"   r#   r1   r%   rL   r   r&   r   r   rJ   r  rM   rN   s   @r*   r   r     s       ) ) ) ) )$ 7;15=A>BDH,1C C|C lC !!23	C
 E-.C  ((9:C !)): ;C !uU->'?!@AC $D>C 
u|	C C C CJ      r)   r   c                   Z    e Zd Z fdZ	 	 	 	 	 	 	 	 	 ddej        dej        deej                 deej                 d	eej                 d
eej                 deeeej                                   dee	         dee	         dee	         dee	         de
eej                 ef         fdZ xZS )BrosEncoderc                     t                                                       | _        t          j        fdt          j                  D                       | _        d S )Nc                 .    g | ]}t                    S r(   )r   ).0_r6   s     r*   
<listcomp>z(BrosEncoder.__init__.<locals>.<listcomp>e  s!    #_#_#_!If$5$5#_#_#_r)   )r0   r1   r6   r   
ModuleListrY   num_hidden_layerslayerrV   s    `r*   r1   zBrosEncoder.__init__b  sV    ]#_#_#_#_uVE]?^?^#_#_#_``


r)   NFTr   r]   r   r   r   r   past_key_values	use_cacher   output_hidden_statesreturn_dictr9   c                    |
rdnd }|	rdnd }|	r| j         j        rdnd }|rdnd }t          | j                  D ]\  }}|
r||fz   }|||         nd }|||         nd }t	          | j         dd          rG| j        r@|rt                              d           d}|                     |j	        |||||||	          }n |||||||||	          }|d         }|r||d         fz  }|	r$||d         fz   }| j         j        r||d	         fz   }|
r||fz   }|st          d
 |||||fD                       S t          |||||          S )Nr(   gradient_checkpointingFzh`use_cache=True` is incompatible with `config.gradient_checkpointing=True`. Setting `use_cache=False`...r   r   r<   r   r;   c              3      K   | ]}||V  	d S rR   r(   )r  vs     r*   	<genexpr>z&BrosEncoder.forward.<locals>.<genexpr>  s4       
 
 =  !===
 
r)   )last_hidden_stater  r   r    cross_attentions)r6   r  	enumerater  r   trainingloggerwarning_gradient_checkpointing_func__call__tupler   )r5   r   r]   r   r   r   r   r  r  r   r  r  all_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cacher\   layer_modulelayer_head_maskr   layer_outputss                        r*   rJ   zBrosEncoder.forwardg  sB    #7@BBD$5?bb4%6d4;;Zdrr`d#,6RR$(44 *	V *	VOA|# I$58H$H!.7.CillO3B3N_Q//TXNt{$<eDD   &NN/   !&I $ A A )! "#)*%	! 	! !-"/!-#1-*?+A#1&7	! 	! 	! *!,M ;"}R'8&::"  V&9]1=M<O&O#;2 V+?=QRCSBU+U( 	E 1]4D D 	 
 
 "&%'(
 
 
 
 
 
 9+.+*1
 
 
 	
r)   )	NNNNNNFFT)r!   r"   r#   r1   r%   rL   r   r&   r   r   r   r   rJ   rM   rN   s   @r*   r  r  a  sV       a a a a a 7;15=A>BEI$(,1/4&*T
 T
|T
 lT
 !!23	T

 E-.T
  ((9:T
 !)): ;T
 "%e.?(@"ABT
 D>T
 $D>T
 'tnT
 d^T
 
uU\"$MM	NT
 T
 T
 T
 T
 T
 T
 T
r)   r  c                   B     e Zd Z fdZdej        dej        fdZ xZS )
BrosPoolerc                     t                                                       t          j        |j        |j                  | _        t          j                    | _        d S rR   )r0   r1   r   rc   r{   r   Tanh
activationrV   s     r*   r1   zBrosPooler.__init__  sC    Yv163EFF
'))r)   r   r9   c                 r    |d d df         }|                      |          }|                     |          }|S )Nr   )r   r8  )r5   r   first_token_tensorpooled_outputs       r*   rJ   zBrosPooler.forward  s@     +111a40

#56666r)   rK   rN   s   @r*   r5  r5    s^        $ $ $ $ $
U\ el        r)   r5  c                   B     e Zd Z fdZdej        dej        fdZ xZS )BrosRelationExtractorc                    t                                                       |j        | _        |j        | _        |j        | _        |j        | _        t          j        | j                  | _	        t          j
        | j        | j        | j        z            | _        t          j
        | j        | j        | j        z            | _        t          j        t          j        d| j                            | _        d S )Nr   )r0   r1   n_relationsr{   backbone_hidden_sizehead_hidden_sizeclassifier_dropout_probr   r   droprc   r   r   	Parameterr%   r   
dummy_noderV   s     r*   r1   zBrosRelationExtractor.__init__  s    !-$*$6! & 2'-'E$Jt;<<	Yt8$:JTMb:bcc
9T68H4K`8`aa,u{1d6O'P'PQQr)   r   r   c           	         |                      |                     |                    }| j                            d                              d|                    d          d          }t          j        ||gd          }|                     |                     |                    }|	                    |                    d          |                    d          | j
        | j                  }|	                    |                    d          |                    d          | j
        | j                  }t          j        |                    dddd          |                    dddd                    }|S )Nr   r   axisr;   r
   )r   rC  rE  	unsqueezerepeatr?   r%   rA   r   r@   r?  rA  r   r   )r5   r   r   	dummy_vecrelation_scores        r*   rJ   zBrosRelationExtractor.forward  sF   jj;!7!788O--a00779>>!;L;LaPP	Iy)41===	HHTYYy1122	!&&Q!1!1!!4!4d6FH]
 
 NN9>>!#4#4innQ6G6GIY[_[pqq	1a++Y->->q!Q-J-J
 
 r)   rK   rN   s   @r*   r=  r=    sc        R R R R R5< EL        r)   r=  c                        e Zd ZdZeZdZd ZdS )BrosPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    brosc                    t          |t          j                  rT|j        j                            d| j        j                   |j         |j        j        	                                 dS dS t          |t          j
                  r_|j        j                            d| j        j                   |j        +|j        j        |j                 	                                 dS dS t          |t          j                  r?|j        j        	                                 |j        j                            d           dS dS )zInitialize the weightsr.   )meanstdNg      ?)r   r   rc   weightdatanormal_r6   initializer_rangera   zero_ry   rn   r   fill_)r5   modules     r*   _init_weightsz!BrosPreTrainedModel._init_weights  s)   fbi(( 	* M&&CT[5R&SSS{& &&((((( '&-- 	*M&&CT[5R&SSS!-"6#56<<>>>>> .--- 	*K""$$$M$$S)))))	* 	*r)   N)r!   r"   r#   r$   r   config_classbase_model_prefixrZ  r(   r)   r*   rN  rN    s:         
 L* * * * *r)   rN  z^The bare Bros Model transformer outputting raw hidden-states without any specific head on top.c            #       @    e Zd Zd fd	Zd Zd Zd Z ee	                    d                     e
ee          	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd	eej                 d
eej                 deej                 deej                 deej                 deej                 deej                 deej                 deej                 deeej                          dee         dee         dee         dee         deeej                 ef         fd                        Z xZS )	BrosModelTc                 (   t                                          |           || _        t          |          | _        t          |          | _        t          |          | _        |rt          |          nd | _
        |                                  d S rR   )r0   r1   r6   rl   r   r_   bbox_embeddingsr  encoderr5  poolerinit_weights)r5   r6   add_pooling_layerr7   s      r*   r1   zBrosModel.__init__  s       ,V441&99"6**,=Gj(((4r)   c                     | j         j        S rR   r   r}   )r5   s    r*   get_input_embeddingszBrosModel.get_input_embeddings  s    ..r)   c                     || j         _        d S rR   rf  )r5   r   s     r*   set_input_embeddingszBrosModel.set_input_embeddings  s    */'''r)   c                     |                                 D ]/\  }}| j        j        |         j                            |           0dS )z
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)itemsra  r  r  r   )r5   heads_to_pruner  r   s       r*   _prune_headszBrosModel._prune_heads!  sU    
 +0022 	C 	CLE5Lu%/;;EBBBB	C 	Cr)   batch_size, sequence_lengthoutput_typer[  Nr   rW   r   rt   rs   r   r   r   r   r  r  r   r  r  r9   c                    ||n| j         j        }||n| j         j        }||n| j         j        }| j         j        r||n| j         j        }nd}||t          d          ||                                }n.||                                dd         }nt          d          |t          d          |\  }}||j        n|j        }|
|
d         d         j	        d         nd}|t          j        ||	          }|gt          | j        d
          r1| j        j        ddd|f         }|                    ||          }|}n!t          j        |t          j        |          }|                     |||          }| j         j        rL|J|                                \  }}}||f}|	t          j        ||	          }	|                     |	          }nd}|                     || j         j                  }|                     |||||          }|j	        d         dk    r|ddddg df         }|| j         j        z  }|                     |          }|                     |||||||
||||          }|d         } | j        |                     |           nd}!|s| |!f|dd         z   S t5          | |!|j        |j        |j        |j                  S )a  
        Returns:

        Examples:

        ```python
        >>> import torch
        >>> from transformers import BrosProcessor, BrosModel

        >>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

        >>> model = BrosModel.from_pretrained("jinho8345/bros-base-uncased")

        >>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
        >>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
        >>> encoding["bbox"] = bbox

        >>> outputs = model(**encoding)
        >>> last_hidden_states = outputs.last_hidden_state
        ```NFzDYou cannot specify both input_ids and inputs_embeds at the same timer<   z5You have to specify either input_ids or inputs_embedszYou have to specify bboxr   r;   )rw   rt   ru   )r   rs   rt   r   r      )r   r   r;   r   r;   r
   r   r
   )
r]   r   r   r   r   r  r  r   r  r  r   )r$  pooler_outputr  r   r    r%  )r6   r   r  use_return_dictr   r  r   r?   rw   r   r%   onesr   r   rt   r   r   r   get_extended_attention_maskinvert_attention_maskget_head_maskr  
bbox_scaler`  ra  rb  r   r  r   r    r%  )"r5   r   rW   r   rt   rs   r   r   r   r   r  r  r   r  r  r   r   r   rw   r   r   r   extended_attention_maskencoder_batch_sizeencoder_sequence_lengthr  encoder_hidden_shapeencoder_extended_attention_maskembedding_outputscaled_bboxbbox_position_embeddingsencoder_outputssequence_outputr;  s"                                     r*   rJ   zBrosModel.forward)  s   N 2C1N--TXT_Tq$8$D  $+Jj 	 &1%<kk$+B];! 	%.%:		@UIII ]%>cddd"#..**KK&',,..ss3KKTUUU<7888!,
J%.%:!!@T DSC^!3A!6!<Q!?!?de!"ZFCCCN!t(899 [*./*HKZK*X'3J3Q3QR\^h3i3i0!A!&[
SY!Z!Z!Z 150P0PQ_alnt0u0u ;! 	3&;&G=R=W=W=Y=Y: 7$68O#P %-).4HQW)X)X)X&.2.H.HI_.`.`++.2+ &&y$+2OPP	??%)'#9 + 
 
 :b>Q11166667DT[33#'#7#7#D#D ,,12"7#B+/!5# ' 
 
 *!,8<8OO444UY 	J#]3oabb6III;-'+;)7&1,=
 
 
 	
r)   )T)NNNNNNNNNNNNNN)r!   r"   r#   r1   rg  ri  rm  r   BROS_INPUTS_DOCSTRINGformatr   r   _CONFIG_FOR_DOCr   r%   rL   r   r&   r   r   r   rJ   rM   rN   s   @r*   r^  r^  
  s       

 
 
 
 
 
/ / /0 0 0C C C +*+@+G+GHe+f+fgg+Wfuvvv -1'+1515/3,0048<9==A$(,0/3&*K
 K
EL)K
 u|$K
 !.	K

 !.K
 u|,K
 EL)K
  -K
  (5K
 !) 6K
 "$u'8"9:K
 D>K
 $D>K
 'tnK
 d^K
  
uU\"$PP	Q!K
 K
 K
 wv hgK
 K
 K
 K
 K
r)   r^  z
    Bros Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                       e Zd ZdgZ fdZ ee                    d                     ee	e
          	 	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 d	eej                 d
eej                 deej                 deej                 deej                 deej                 dee         dee         dee         deeej                 e	f         fd                        Z xZS )BrosForTokenClassificationrb  c                 h   t                                          |           |j        | _        t          |          | _        t          |d          r|j        n|j        }t          j	        |          | _
        t          j        |j        |j                  | _        |                                  d S Nclassifier_dropout)r0   r1   
num_labelsr^  rO  r   r  r   r   r   r   rc   r{   
classifierrc  r5   r6   r  r7   s      r*   r1   z#BrosForTokenClassification.__init__  s        +f%%	)09M)N)NnF%%TZTn 	 z"455)F$68IJJr)   rn  ro  Nr   rW   r   bbox_first_token_maskrt   rs   r   r   labelsr   r  r  r9   c                 ~   ||n| j         j        }|                     ||||||||
||
  
        }|d         }|                     |          }|                     |          }d}|	t                      }|Z|                    d          } ||                    d| j                  |         |	                    d          |                   }n8 ||                    d| j                  |	                    d                    }|s|f|dd         z   }||f|z   n|S t          |||j	        |j
                  S )ax  

        Returns:

        Examples:

        ```python
        >>> import torch
        >>> from transformers import BrosProcessor, BrosForTokenClassification

        >>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

        >>> model = BrosForTokenClassification.from_pretrained("jinho8345/bros-base-uncased")

        >>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
        >>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
        >>> encoding["bbox"] = bbox

        >>> outputs = model(**encoding)
        ```N)	rW   r   rt   rs   r   r   r   r  r  r   r<   r;   r   logitsr   r    )r6   rt  rO  r   r  r	   r@   r  r   r   r    )r5   r   rW   r   r  rt   rs   r   r   r  r   r  r  r   r  r  r   loss_fctr   s                      r*   rJ   z"BrosForTokenClassification.forward  st   L &1%<kk$+B]))))%'/!5#  
 
 "!*,,7711'))H$0(=(B(B2(F(F%xKKDO445JKV[[Y[__]rMs   xB @ @&++b//RR 	FY,F)-)9TGf$$vE$!/)	
 
 
 	
r)   NNNNNNNNNNNNr!   r"   r#   "_keys_to_ignore_on_load_unexpectedr1   r   r  r  r   r   r  r   r%   rL   r   r   r   rJ   rM   rN   s   @r*   r  r    s        +4&     +*+@+G+GHe+f+fgg+@___ -1'+158<15/3,004)-,0/3&*L
 L
EL)L
 u|$L
 !.	L

  (5L
 !.L
 u|,L
 EL)L
  -L
 &L
 $D>L
 'tnL
 d^L
 
uU\"$99	:L
 L
 L
 `_ hgL
 L
 L
 L
 L
r)   r  a  
    Bros Model with a token classification head on top (initial_token_layers and subsequent_token_layer on top of the
    hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. The initial_token_classifier is used to
    predict the first token of each entity, and the subsequent_token_classifier is used to predict the subsequent
    tokens within an entity. Compared to BrosForTokenClassification, this model is more robust to serialization errors
    since it predicts next token from one token.
    c            !           e Zd ZdgZ fdZ ee                    d                     ee	e
          	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 d	eej                 d
eej                 deej                 deej                 deej                 deej                 deej                 dee         dee         dee         deeej                 e	f         fd                        Z xZS )!BrosSpadeEEForTokenClassificationrb  c           	      H   t                                          |           || _        |j        | _        |j        | _        |j        | _        t          |          | _        t          |d          r|j
        n|j        }t          j        t          j        |          t          j        |j        |j                  t          j        |          t          j        |j        |j                            | _        t#          |          | _        |                                  d S r  )r0   r1   r6   r  r?  r{   r@  r^  rO  r   r  r   r   
Sequentialr   rc   initial_token_classifierr=  subsequent_token_classifierrc  r  s      r*   r1   z*BrosSpadeEEForTokenClassification.__init__.  s        +!-$*$6!f%%	)09M)N)NnF%%TZTn 	
 )+J)**If(&*<==J)**If(&*;<<	)
 )
% ,A+H+H(r)   rn  ro  Nr   rW   r   r  rt   rs   r   r   initial_token_labelssubsequent_token_labelsr   r  r  r9   c                 L   ||n| j         j        }|                     ||||||||||
  
        }|d         }|                    dd                                          }|                     |                              dd                                          }|                     ||                              d          }d|z
  }|j        \  }}|j	        }t          j        |t          j        |dg                              |          gd                                          }|                    |dddddf         t          j        |j                  j                  }t          j        ||dz                                 |                                          }|                    |dddddf         t          j        |j                  j                  }|                    d                                          }d}|	|
t+                      }|	                    d          }	|G|                    d          } ||                    d| j                  |         |	|                   }n% ||                    d| j                  |	          }|
                    d          }
 ||                    d|dz             |         |
|                   }||z   }|s||f|dd         z   }||f|z   n|S t/          ||||j        |j                  S )	a  
        Returns:

        Examples:

        ```python
        >>> import torch
        >>> from transformers import BrosProcessor, BrosSpadeEEForTokenClassification

        >>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

        >>> model = BrosSpadeEEForTokenClassification.from_pretrained("jinho8345/bros-base-uncased")

        >>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
        >>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
        >>> encoding["bbox"] = bbox

        >>> outputs = model(**encoding)
        ```N
r   rW   r   rt   rs   r   r   r   r  r  r   r   rG  r<   r;   )r   r   r   r   r    )r6   rt  rO  rh   r   r  r  squeezer   rw   r%   rA   r   r   r   masked_fillfinforv   mineyer@   r	   r  r   r   r    )r5   r   rW   r   r  rt   rs   r   r   r  r  r   r  r  r   last_hidden_statesr   r   inv_attention_maskr   max_seq_lengthrw   invalid_token_maskself_token_masksubsequent_token_maskr   r  initial_token_losssubsequent_token_lossr   s                                 r*   rJ   z)BrosSpadeEEForTokenClassification.forwardG  s{   L &1%<kk$+B]))))%'/!5#  
 
 %QZ/99!Q??JJLL#<<=OPPZZ[\^_``kkmm"&"B"BCUWi"j"j"r"rst"u"u /%7%="
N#*"Y(:EKUV<X<X<[<[\b<c<c'dklmmmrrtt"9"E"Eqqq$z*EK8O8U,V,V,Z#
 #
  )NNQ4FGGJJ6RRWWYY"9"E"ED!!!QQQJ'5L5R)S)S)W#
 #
 !/ 3 3B 7 7 < < > >+0G0S'))H $8#<#<R#@#@ $0(=(B(B2(F(F%%-X(--b$/BBCXY()>?& &""
 &.X.B.G.GDO.\.\^r%s%s"&=&B&B2&F&F#$,H',,R!1CDDEZ['(=>% %!
 &(==D 	F*,CDwqrr{RF)-)9TGf$$vE!5$;!/)
 
 
 	
r)   )NNNNNNNNNNNNN)r!   r"   r#   r  r1   r   r  r  r   r   r  r   r%   rL   r   r   r   rJ   rM   rN   s   @r*   r  r  !  s        +4&    2 +*+@+G+GHe+f+fgg?YYY -1'+158<15/3,0047;:>,0/3&*g
 g
EL)g
 u|$g
 !.	g

  (5g
 !.g
 u|,g
 EL)g
  -g
 'u|4g
 "*%,!7g
 $D>g
 'tng
 d^g
 
uU\"O3	4g
 g
 g
 ZY hgg
 g
 g
 g
 g
r)   r  z
    Bros Model with a token classification head on top (a entity_linker layer on top of the hidden-states output) e.g.
    for Entity-Linking. The entity_linker is used to predict intra-entity links (one entity to another entity).
    c                       e Zd ZdgZ fdZ ee                    d                     ee	e
          	 	 	 	 	 	 	 	 	 	 	 	 ddeej                 deej                 deej                 d	eej                 d
eej                 deej                 deej                 deej                 deej                 dee         dee         dee         deeej                 e	f         fd                        Z xZS )!BrosSpadeELForTokenClassificationrb  c                 T   t                                          |           || _        |j        | _        |j        | _        |j        | _        t          |          | _        t          |d          r|j
        n|j         t          |          | _        |                                  d S r  )r0   r1   r6   r  r?  r{   r@  r^  rO  r   r  r   r=  entity_linkerrc  rV   s     r*   r1   z*BrosSpadeELForTokenClassification.__init__  s        +!-$*$6!f%%	&-f6J&K&K	k	"	"QWQk26::r)   rn  ro  Nr   rW   r   r  rt   rs   r   r   r  r   r  r  r9   c                    ||n| j         j        }|                     ||||||||
||
  
        }|d         }|                    dd                                          }|                     ||                              d          }d}|	}t                      }|j        \  }}|j	        }t          j        ||dz                                 |                                          }|                    d          }t          j        | t          j        |dgt          j                                      |          gd          }|                    |dddddf         t          j        |j                  j                  }|                    |dddddf         t          j        |j                  j                  } ||                    d|dz             |         |	                    d          |                   }|s|f|dd         z   }||f|z   n|S t+          |||j        |j        	          S )
a  
        Returns:

        Examples:

        ```python
        >>> import torch
        >>> from transformers import BrosProcessor, BrosSpadeELForTokenClassification

        >>> processor = BrosProcessor.from_pretrained("jinho8345/bros-base-uncased")

        >>> model = BrosSpadeELForTokenClassification.from_pretrained("jinho8345/bros-base-uncased")

        >>> encoding = processor("Hello, my dog is cute", add_special_tokens=False, return_tensors="pt")
        >>> bbox = torch.tensor([[[0, 0, 1, 1]]]).repeat(1, encoding["input_ids"].shape[-1], 1)
        >>> encoding["bbox"] = bbox

        >>> outputs = model(**encoding)
        ```Nr  r   r   r<   r   rG  r;   r  )r6   rt  rO  rh   r   r  r  r	   r   rw   r%   r  r   r   r@   rA   r   r  r  rv   r  r   r   r    )r5   r   rW   r   r  rt   rs   r   r   r  r   r  r  r   r  r  r   r  r   r  rw   r  maskr   s                           r*   rJ   z)BrosSpadeELForTokenClassification.forward  si   H &1%<kk$+B]))))%'/!5#  
 
 %QZ/99!Q??JJLL##$68JKKSSTUVV'))H)7)=&J#*F#i8JKKNNvVV[[]]O(--b11D$)I**KQuzBBBEEfMM % % %! ''(=aaaqqqj(I5;W]WcKdKdKhiiF''aaa
(CU[QWQ]E^E^EbccF8FKKNQ,>??Ev{{SUW[G\]]D 	FY,F)-)9TGf$$vE$!/)	
 
 
 	
r)   r  r  rN   s   @r*   r  r    s        +4&     +*+@+G+GHe+f+fgg+@___ -1'+158<15/3,004)-,0/3&*U
 U
EL)U
 u|$U
 !.	U

  (5U
 !.U
 u|,U
 EL)U
  -U
 &U
 $D>U
 'tnU
 d^U
 
uU\"$99	:U
 U
 U
 `_ hgU
 U
 U
 U
 U
r)   r  )=r$   r   dataclassesr   typingr   r   r   r   r%   torch.utils.checkpointr   torch.nnr	   activationsr   modeling_outputsr   r   r   modeling_utilsr   pytorch_utilsr   r   r   utilsr   r   r   r   r   configuration_brosr   
get_loggerr!   r(  _CHECKPOINT_FOR_DOCr  BROS_START_DOCSTRINGr  r   Moduler,   rP   r_   rl   r   r   r   r   r   r   r  r5  r=  rN  r^  r  r  r  r(   r)   r*   <module>r     s      ! ! ! ! ! ! / / / / / / / / / / / /            % % % % % % ! ! ! ! ! !         
 . - - - - - l l l l l l l l l l              + * * * * * 
	H	%	%3 	 A H : : : : :k : : :>    	   *    	   &       ? ? ? ? ? ? ? ?DG G G G G	 G G GV    RY   3 3 3 3 3BI 3 3 3n    ry          W W W W W	 W W WtZ
 Z
 Z
 Z
 Z
") Z
 Z
 Z
|           BI   D* * * * */ * * *4 d h
 h
 h
 h
 h
# h
 h
	 h
V   ^
 ^
 ^
 ^
 ^
!4 ^
 ^
 ^
B  	 	E
 E
 E
 E
 E
(; E
 E
	 	E
P   h
 h
 h
 h
 h
(; h
 h
 h
 h
 h
r)   