
    gP;                     J   d Z ddlZddlmZmZmZmZmZ ddlm	Z	 ddl
mZmZmZ ddlmZmZmZmZ ddlmZmZmZ dd	lmZ erdd
lmZ  ej        e          ZdefdZd Zd Z d Z!d Z" G d ded          Z# G d ded          Z$e#e$j%        d<    G d de          Z&dS )z
Processor class for Idefics3.
    N)TYPE_CHECKINGDictListOptionalUnion   )BatchFeature)
ImageInputis_valid_image
load_image)ImagesKwargsProcessingKwargsProcessorMixinUnpack)
AddedTokenBatchEncoding	TextInput)logging)PreTokenizedInputreturnc                 V    t          | t                    o|                     d          S )Nhttp)
isinstancestr
startswith)vals    l/var/www/html/ai-engine/env/lib/python3.11/site-packages/transformers/models/idefics3/processing_idefics3.pyis_urlr   #   s#    c3:CNN6$:$::    c                 >    t          |           pt          |           S N)r   r   )elems    r   is_image_or_image_urlr#   '   s    $<</>$///r   c           	          d}t          |          D ]6}t          |          D ]}|| d|dz    d|dz    dz   | | z  z   z  } |dz  }7|d| | z   | | z  z   | z   z  }|S )zKPrompt with expanded image tokens for when the image is split into patches. z<row_   _col_>
)range)	image_seq_len
image_rows
image_colsfake_token_around_imageimage_tokenglobal_img_tokentext_split_imagesn_hn_ws	            r   _prompt_split_imager4   +   s    Z   " "$$ 	 	C*,/OsQw/O/OS1W/O/O/OOU`RberRrr 	T!&$&&
	 
]
*	+ %
&	' r   c                 &    | | z   | | z  z   | z   S )z5Prompt with expanded image tokens for a single image. )r+   r.   r/   r0   s       r   _prompt_single_imager7   >   s7     #$
	 
]
*	+ %
&	'r   c                 h    | dk    r|dk    rt          ||||          S t          || ||||          S )Nr   )r.   r/   r0   )r7   r4   )r,   r-   r+   r.   r/   r0   s         r   get_image_prompt_stringr9   H   s[     Q:??#$;#-	
 
 
 	
 z:/FUe  r   c                   L    e Zd ZU ee         ed<   eeeef                  ed<   dS )Idefics3ImagesKwargsreturn_row_col_infomax_image_sizeN)	__name__
__module____qualname__r   bool__annotations__r   r   intr6   r   r   r;   r;   W   s;         !$'''T#s(^,,,,,,r   r;   F)totalc                   0    e Zd ZU eed<   ddddddidZdS )Idefics3ProcessorKwargsimages_kwargsTF)add_special_tokenspaddingis_split_into_wordsr<   )text_kwargsrG   N)r>   r?   r@   r;   rB   	_defaultsr6   r   r   rF   rF   \   sI         '''' #'#(
 
 "4
	 	IIIr   rF   rG   c                       e Zd ZdZddgZdZdZdded	ef fd
Z	d Z
	 	 	 	 	 ddeeee         eee                  f         deedee         ed         f         dee         dee         def
dZd Zd Zed             Z xZS )Idefics3Processora  
    Constructs a Idefics3 processor which wraps a LLama tokenizer and Idefics3 image processor into a single processor.

    [`Idefics3Processor`] offers all the functionalities of [`Idefics3ImageProcessor`] and [`Idefics3TokenizerFast`]. See
    the docstring of [`~IdeficsProcessor.__call__`] and [`~IdeficsProcessor.decode`] for more information.

    Args:
        image_processor (`Idefics3ImageProcessor`):
            An instance of [`Idefics3ImageProcessor`]. The image processor is a required input.
        tokenizer (`PreTrainedTokenizerBase`, *optional*):
            An instance of [`PreTrainedTokenizerBase`]. This should correspond with the model's text model. The tokenizer is a required input.
        image_seq_len (`int`, *optional*, defaults to 169):
            The length of the image sequence i.e. the number of <image> tokens per image in the input.
            This parameter is used to build the string from the input prompt and image tokens and should match the
            value the model used. It is computed as: image_seq_len = int(((image_size // patch_size) ** 2) / (scale_factor**2))
        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
            in a chat into a tokenizable string.
    image_processor	tokenizerIdefics3ImageProcessorAutoTokenizerN   r+   chat_templatec                    |t          d          |t          d          t          ddd          | _        t          ddd          | _        t          ddd          | _        d	| _        || _        t          j        d
          | _	        d| j        | j        | j        gi}|
                    |            t                      j        ||fd|i| d S )Nz)You need to specify an `image_processor`.z"You need to specify a `tokenizer`.z<fake_token_around_image>FT)
normalizedspecial<image>z<end_of_utterance>z<global-img>z*(\n?<global-img>\n?|<row_\d+_col_\d+>\n?)+additional_special_tokensrT   )
ValueErrorr   fake_image_tokenr/   end_of_utterance_tokenglobal_image_tagr+   recompile%_regex_to_remove_extra_special_tokensrH   super__init__)selfrO   rP   r+   rT   kwargstokens_to_add	__class__s          r   rb   zIdefics3Processor.__init__   s   "HIIIABBB *+FSXbf g g g%iE4PPP&01ERWae&f&f&f# .* 68Z@m5n5n2 (% +*
 	$$]333)[[=[TZ[[[[[r   c                     g }|D ]t}g }|D ]X}t          |          r|                    |           't          |          r"|                    t          |                     Y|                    |           u|S r!   )r   appendr   r   )rc   promptsprompt_imagespromptimagesr"   s         r   _extract_images_from_promptsz.Idefics3Processor._extract_images_from_prompts   s     	) 	)FF 4 4!$'' 4MM$''''D\\ 4MM*T"2"2333  ((((r   rl   textr   rd   r   c           
      ~   ||t          d           | j        t          fd| j        j        i|}||n| j        }g }g }	t                      }
|t          |          r|gg}nt          |t                    rt          |d                   r|g}nZt          |t                    sEt          |d         t                    s*t          |d         d                   st          d          d |D             }	d |D             } | j
        |fi |d         }|
                    |           |t          |t                    r|g}n?t          |t                    s*t          |d         t                    st          d	          |
                    d
dgt          |          z  g          }|
                    ddgt          |          z  g          }| j        j        }| j        j        }| j        }g }t'          |||          D ]\  }}}|                    |                    |                     g }t'          ||          D ]/\  }}t-          ||||||          }|                    |           0|                    |          }t          |          dk    rt          d          |d         }t1          |          D ]\  }}||||dz            z   z  }|                    |            | j        dd|i|d         }|
                    |           |	|k    rt          d| d|	 d          |
S )a  
        Processes the input prompts and returns a BatchEncoding.

        Example:

        ```python
        >>> import requests
        >>> from transformers import Idefics3Processor
        >>> from transformers.image_utils import load_image

        >>> processor = Idefics3Processor.from_pretrained("HuggingFaceM4/Idefics3-8B-Llama3")
        >>> processor.image_processor.do_image_splitting = False  # Force as False to simplify the example

        >>> url1 = "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
        >>> url2 = "https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg"

        >>> image1, image2 = load_image(url1), load_image(url2)
        >>> images = [[image1], [image2]]

        >>> text = [
        ...     "<image>In this image, we see",
        ...     "bla bla bla<image>",
        ... ]
        >>> outputs = processor(images=images, text=text, return_tensors="pt", padding=True)
        >>> input_ids = outputs.input_ids
        >>> input_tokens = processor.tokenizer.batch_decode(input_ids)
        >>> print(input_tokens)
        ['<|begin_of_text|><fake_token_around_image><global-img>((<image>)*169)<fake_token_around_image> In this image, we see', '<|reserved_special_token_0|><|reserved_special_token_0|><|reserved_special_token_0|><|begin_of_text|>bla bla bla<fake_token_around_image><global-img>((<image>)*169)<fake_token_around_image>']
        ```

        Args:
            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`, *optional*):
                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
                tensor. If is of type `List[ImageInput]`, it's assumed that this is for a single prompt i.e. of batch size 1.
            text (`Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]`, *optional*):
                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
                Wherever an image token, `<image>` is encountered it is expanded to
                `<fake_token_around_image>` + `<row_x_col_y>` + `<image>` * `image_seq_len` * <fake_token_around_image>`.
            image_seq_len (`int`, *optional*):
                The length of the image sequence. If not provided, the default value of self.image_seq_len is used.
                image_seq_len should be equal to int(((image_size // patch_size) ** 2) / (scale_factor**2))
            return_tensors (`Union[str, TensorType]`, *optional*):
                If set, will return tensors of a particular framework. See [`PreTrainedTokenizerFast.__call__`] for more
                information.
        Nz+You must provide either `text` or `images`.tokenizer_init_kwargsr   zdInvalid input images. Please provide a single image or a list of images or a list of list of images.c                 ,    g | ]}t          |          S r6   )len.0samples     r   
<listcomp>z.Idefics3Processor.__call__.<locals>.<listcomp>  s    !C!C!C&#f++!C!C!Cr   c                 &    g | ]}d  |D             S )c                 N    g | ]"}t          |          rt          |          n|#S r6   )r   r   )rt   ims     r   rv   z9Idefics3Processor.__call__.<locals>.<listcomp>.<listcomp>  s-    MMM;z"~~~MMMr   r6   rs   s     r   rv   z.Idefics3Processor.__call__.<locals>.<listcomp>  s(    cccRXMMfMMMcccr   rG   zAInvalid input text. Please provide a string, or a list of stringsrowscols)r/   r.   r0   z.The image token should be present in the text.r&   rn   rK   z!The number of images in the text z and images  z should be the same.r6   )rZ   _merge_kwargsrF   rP   init_kwargsr+   r	   r#   r   listrO   updater   poprr   r[   contentr/   r]   ziprh   countr9   split	enumerate)rc   rl   rn   audiovideosr+   rd   output_kwargsn_images_in_textn_images_in_imagesinputsimage_inputsr,   r-   r[   r/   r0   prompt_stringsru   sample_rowssample_colsimage_prompt_stringsn_rowsn_colsimage_prompt_stringsplit_sampleitext_inputss                               r   __call__zIdefics3Processor.__call__   s   p <FNJKKK**#
 
"&."<
 
 
 *7)BHZ$V,, !(FD)) 	.CF1I.N.N 	 vt,,"6!9d33 .fQil;;
 !z   "D!CF!C!C!C dc\bcccF/4/YY-:XYYLMM,'''$$$ fvd++ fJtAw4L4L f !deeeFaS3t99_,=>>JFaS3t99_,=>>J#4<*2K#4N47j*4U4U . .0[ ''[(A(ABBB (*$&)+{&C&C 	E 	ENFF*A%$/0@)9+ + +' )//0CDDDD%||K88|$$))$%UVVV &a.78L.M.M H H*A*1LQ4GGGFF%%f----($.]]n]m@\]]KMM+&&&!%555 8HWi   r   c                 D       j         j        |i |} fd|D             S )z
        This method forwards all its arguments to Idefics3TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
        refer to the docstring of this method for more information.
        c                 F    g | ]}j                             d |          S )rX   )r`   sub)rt   src   s     r   rv   z2Idefics3Processor.batch_decode.<locals>.<listcomp>A  s-    lllQR:>>y!LLlllr   )rP   batch_decode)rc   argsrd   batched_decode_outputs   `   r   r   zIdefics3Processor.batch_decode;  s:    
 !< ;T LV L LllllVkllllr   c                 \     | j         j        |i |}| j                            d|          S )z
        This method forwards all its arguments to Idefics3TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
        the docstring of this method for more information.
        rX   )rP   decoder`   r   )rc   r   rd   decode_outputs       r   r   zIdefics3Processor.decodeC  s6    
 .-t>v>>9==iWWWr   c                     | j         j        }| j        j        }t          t                              ||z                       S r!   )rP   model_input_namesrO   r~   dictfromkeys)rc   tokenizer_input_namesimage_processor_input_namess      r   r   z#Idefics3Processor.model_input_namesK  s:     $ @&*&:&L#DMM"7:U"UVVWWWr   )NrS   N)NNNNN)r>   r?   r@   __doc__
attributesimage_processor_classtokenizer_classrC   r   rb   rm   r   r
   r   r   r   r   rF   r   r   r   r   propertyr   __classcell__)rf   s   @r   rN   rN   n   su        & $[1J4%O\ \s \ad \ \ \ \ \ \6
 
 
 OSbf'+L Lj$z"2Dj9I4JJKL I2DOTJ]E^^_L  }L 01L 
L L L L\m m mX X X X X XX X X X Xr   rN   )'r   r^   typingr   r   r   r   r   feature_extraction_utilsr	   image_utilsr
   r   r   processing_utilsr   r   r   r   tokenization_utils_baser   r   r   utilsr   r   
get_loggerr>   loggerrA   r   r#   r4   r7   r9   r;   rF   rB   rN   r6   r   r   <module>r      s	    
			 = = = = = = = = = = = = = = 4 4 4 4 4 4 A A A A A A A A A A V V V V V V V V V V V V K K K K K K K K K K        =<<<<<<		H	%	%;4 ; ; ; ;0 0 0  &    - - - - -<u - - - -
    .e     <P  ' 8aX aX aX aX aX aX aX aX aX aXr   