




'''simple docstring'''



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

UpperCAmelCase_    :					List[Any]					      =						logging.get_logger(__name__)

UpperCAmelCase_    :					Union[str, Any]					      =						{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase_    :					Union[str, Any]					      =						{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

UpperCAmelCase_    :					Union[str, Any]					      =						{
    '''gpt2''': 1_0_2_4,
    '''gpt2-medium''': 1_0_2_4,
    '''gpt2-large''': 1_0_2_4,
    '''gpt2-xl''': 1_0_2_4,
    '''distilgpt2''': 1_0_2_4,
}







@lru_cache()
def      _UpperCamelCase	()->				Optional[int]:




     '''simple docstring'''





     __snake_case         =					(
         list(range(ord('''!'''		)				,		ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)				,		ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)				,		ord('''ÿ'''		) + 1		)		)
     )
     __snake_case         =					bs[:]
     __snake_case         =					0
     for b in range(2**8		):
          if b not in bs:
               bs.append(_lowerCamelCase		)
               cs.append(2**8 + n		)
               n += 1
     __snake_case         =					[chr(_lowerCamelCase		) for n in cs]
     return dict(zip(_lowerCamelCase				,		_lowerCamelCase		)		)







def      _UpperCamelCase	(_lowerCamelCase							:       str		)->				Tuple:




     '''simple docstring'''





     __snake_case         =					set()
     __snake_case         =					word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)		)
          __snake_case         =					char
     return pairs







class       lowerCAmelCase				(	__lowerCAmelCase):
       __lowercase					: Optional[Any]	    =     VOCAB_FILES_NAMES
       __lowercase					: Dict	    =     PRETRAINED_VOCAB_FILES_MAP
       __lowercase					: Tuple	    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowercase					: Optional[int]	    =     ['''input_ids''', '''attention_mask''']


       def __init__(				self      ,					__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE="replace"      ,					__SCREAMING_SNAKE_CASE="<|endoftext|>"      ,					__SCREAMING_SNAKE_CASE="<|endoftext|>"      ,					__SCREAMING_SNAKE_CASE="<|endoftext|>"      ,					__SCREAMING_SNAKE_CASE=None      ,					__SCREAMING_SNAKE_CASE=False      ,					__SCREAMING_SNAKE_CASE=False      ,					**__SCREAMING_SNAKE_CASE      ,					) ->					List[str]:



            '''simple docstring'''




            __snake_case         =					AddedToken(__SCREAMING_SNAKE_CASE      ,					lstrip=__SCREAMING_SNAKE_CASE      ,					rstrip=__SCREAMING_SNAKE_CASE					) if isinstance(__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE					) else bos_token
            __snake_case         =					AddedToken(__SCREAMING_SNAKE_CASE      ,					lstrip=__SCREAMING_SNAKE_CASE      ,					rstrip=__SCREAMING_SNAKE_CASE					) if isinstance(__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE					) else eos_token
            __snake_case         =					AddedToken(__SCREAMING_SNAKE_CASE      ,					lstrip=__SCREAMING_SNAKE_CASE      ,					rstrip=__SCREAMING_SNAKE_CASE					) if isinstance(__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE					) else unk_token
            __snake_case         =					AddedToken(__SCREAMING_SNAKE_CASE      ,					lstrip=__SCREAMING_SNAKE_CASE      ,					rstrip=__SCREAMING_SNAKE_CASE					) if isinstance(__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE					) else pad_token
            super().__init__(
                errors=__SCREAMING_SNAKE_CASE      ,					unk_token=__SCREAMING_SNAKE_CASE      ,					bos_token=__SCREAMING_SNAKE_CASE      ,					eos_token=__SCREAMING_SNAKE_CASE      ,					pad_token=__SCREAMING_SNAKE_CASE      ,					add_prefix_space=__SCREAMING_SNAKE_CASE      ,					add_bos_token=__SCREAMING_SNAKE_CASE      ,					**__SCREAMING_SNAKE_CASE      ,					)
            __snake_case         =					add_bos_token

            with open(__SCREAMING_SNAKE_CASE      ,					encoding='''utf-8'''					) as vocab_handle:
                 __snake_case         =					json.load(__SCREAMING_SNAKE_CASE					)
            __snake_case         =					{v: k for k, v in self.encoder.items()}
            __snake_case         =					errors  # how to handle errors in decoding
            __snake_case         =					bytes_to_unicode()
            __snake_case         =					{v: k for k, v in self.byte_encoder.items()}
            with open(__SCREAMING_SNAKE_CASE      ,					encoding='''utf-8'''					) as merges_handle:
                 __snake_case         =					merges_handle.read().split('''\n'''					)[1:-1]
            __snake_case         =					[tuple(merge.split()					) for merge in bpe_merges]
            __snake_case         =					dict(zip(__SCREAMING_SNAKE_CASE      ,					range(len(__SCREAMING_SNAKE_CASE					)					)					)					)
            __snake_case         =					{}
            __snake_case         =					add_prefix_space

            # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
            __snake_case         =					re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''					)


       @property
       def  lowerCAmelCase (				self					) ->					Optional[Any]:



            '''simple docstring'''




            return len(self.encoder					)


       def  lowerCAmelCase (				self					) ->					Union[str, Any]:



            '''simple docstring'''




            return dict(self.encoder      ,					**self.added_tokens_encoder					)


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE					) ->					Optional[int]:



            '''simple docstring'''




            if token in self.cache:
                 return self.cache[token]
            __snake_case         =					tuple(__SCREAMING_SNAKE_CASE					)
            __snake_case         =					get_pairs(__SCREAMING_SNAKE_CASE					)

            if not pairs:
                 return token

            while True:
                 __snake_case         =					min(__SCREAMING_SNAKE_CASE      ,					key=lambda __SCREAMING_SNAKE_CASE					: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE      ,					float('''inf'''					)					)					)
                 if bigram not in self.bpe_ranks:
                      break
                 __snake_case   ,      __snake_case         =					bigram
                 __snake_case         =					[]
                 __snake_case         =					0
                 while i < len(__SCREAMING_SNAKE_CASE					):
                      try:
                           __snake_case         =					word.index(__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE					)
                      except ValueError:
                           new_word.extend(word[i:]					)
                           break
                      else:
                           new_word.extend(word[i:j]					)
                           __snake_case         =					j

                      if word[i] == first and i < len(__SCREAMING_SNAKE_CASE					) - 1 and word[i + 1] == second:
                           new_word.append(first + second					)
                           i += 2
                      else:
                           new_word.append(word[i]					)
                           i += 1
                 __snake_case         =					tuple(__SCREAMING_SNAKE_CASE					)
                 __snake_case         =					new_word
                 if len(__SCREAMING_SNAKE_CASE					) == 1:
                      break
                 else:
                      __snake_case         =					get_pairs(__SCREAMING_SNAKE_CASE					)
            __snake_case         =					''' '''.join(__SCREAMING_SNAKE_CASE					)
            __snake_case         =					word
            return word


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE=None					) ->					Tuple:



            '''simple docstring'''




            if self.add_bos_token:
                 __snake_case         =					[self.bos_token_id]
            else:
                 __snake_case         =					[]

            __snake_case         =					bos_token_ids + token_ids_a

            if token_ids_a is None:
                 return output

            return output + bos_token_ids + token_ids_a


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE = None      ,					__SCREAMING_SNAKE_CASE = False					) ->					List[int]:



            '''simple docstring'''




            if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__SCREAMING_SNAKE_CASE      ,					token_ids_a=__SCREAMING_SNAKE_CASE      ,					already_has_special_tokens=__SCREAMING_SNAKE_CASE					)

            if not self.add_bos_token:
                 return super().get_special_tokens_mask(
                     token_ids_a=__SCREAMING_SNAKE_CASE      ,					token_ids_a=__SCREAMING_SNAKE_CASE      ,					already_has_special_tokens=__SCREAMING_SNAKE_CASE					)

            if token_ids_a is None:
                 return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					))
            return [1] + ([0] * len(__SCREAMING_SNAKE_CASE					)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE					))


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE					) ->					Dict:



            '''simple docstring'''




            __snake_case         =					[]
            for token in re.findall(self.pat      ,					__SCREAMING_SNAKE_CASE					):
                 __snake_case         =					''''''.join(
                     self.byte_encoder[b] for b in token.encode('''utf-8'''					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                 bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__SCREAMING_SNAKE_CASE					).split(''' '''					)					)
            return bpe_tokens


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE					) ->					List[str]:



            '''simple docstring'''




            return self.encoder.get(__SCREAMING_SNAKE_CASE      ,					self.encoder.get(self.unk_token					)					)


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE					) ->					str:



            '''simple docstring'''




            return self.decoder.get(__SCREAMING_SNAKE_CASE					)


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE					) ->					List[Any]:



            '''simple docstring'''




            __snake_case         =					''''''.join(__SCREAMING_SNAKE_CASE					)
            __snake_case         =					bytearray([self.byte_decoder[c] for c in text]					).decode('''utf-8'''      ,					errors=self.errors					)
            return text


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE = None					) ->					Tuple[str]:



            '''simple docstring'''




            if not os.path.isdir(__SCREAMING_SNAKE_CASE					):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
                 return
            __snake_case         =					os.path.join(
                __SCREAMING_SNAKE_CASE      ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
            __snake_case         =					os.path.join(
                __SCREAMING_SNAKE_CASE      ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)

            with open(__SCREAMING_SNAKE_CASE      ,					'''w'''      ,					encoding='''utf-8'''					) as f:
                 f.write(json.dumps(self.encoder      ,					indent=2      ,					sort_keys=__SCREAMING_SNAKE_CASE      ,					ensure_ascii=__SCREAMING_SNAKE_CASE					) + '''\n'''					)

            __snake_case         =					0
            with open(__SCREAMING_SNAKE_CASE      ,					'''w'''      ,					encoding='''utf-8'''					) as writer:
                 writer.write('''#version: 0.2\n'''					)
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,					key=lambda __SCREAMING_SNAKE_CASE					: kv[1]					):
                      if index != token_index:
                           logger.warning(
                               F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               ''' Please check that the tokenizer is not corrupted!'''					)
                           __snake_case         =					token_index
                      writer.write(''' '''.join(__SCREAMING_SNAKE_CASE					) + '''\n'''					)
                      index += 1

            return vocab_file, merge_file


       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE=False      ,					**__SCREAMING_SNAKE_CASE					) ->					List[Any]:



            '''simple docstring'''




            __snake_case         =					kwargs.pop('''add_prefix_space'''      ,					self.add_prefix_space					)
            if is_split_into_words or add_prefix_space:
                 __snake_case         =					''' ''' + text
            return (text, kwargs)




       def  lowerCAmelCase (				self      ,					__SCREAMING_SNAKE_CASE					) ->					List[int]:



            '''simple docstring'''




            __snake_case         =					[]
            for is_user, text in conversation.iter_texts():
                 input_ids.extend(self.encode(__SCREAMING_SNAKE_CASE      ,					add_special_tokens=__SCREAMING_SNAKE_CASE					) + [self.eos_token_id]					)
            if len(__SCREAMING_SNAKE_CASE					) > self.model_max_length:
                 __snake_case         =					input_ids[-self.model_max_length :]
            return input_ids












'''simple docstring'''



import inspect

from diffusers.utils import is_flax_available
from diffusers.utils.testing_utils import require_flax


if is_flax_available():
     import jax





@require_flax
class       lowerCAmelCase				:


       def  lowerCAmelCase (				self					) ->					int:



            '''simple docstring'''




            __snake_case   ,      __snake_case         =					self.prepare_init_args_and_inputs_for_common()

            __snake_case         =					self.model_class(**__SCREAMING_SNAKE_CASE					)
            __snake_case         =					model.init(inputs_dict['''prng_key''']      ,					inputs_dict['''sample''']					)
            jax.lax.stop_gradient(__SCREAMING_SNAKE_CASE					)

            __snake_case         =					model.apply(__SCREAMING_SNAKE_CASE      ,					inputs_dict['''sample''']					)

            if isinstance(__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE					):
                 __snake_case         =					output.sample

            self.assertIsNotNone(__SCREAMING_SNAKE_CASE					)
            __snake_case         =					inputs_dict['''sample'''].shape
            self.assertEqual(output.shape      ,					__SCREAMING_SNAKE_CASE      ,					'''Input and output shapes do not match'''					)


       def  lowerCAmelCase (				self					) ->					Tuple:



            '''simple docstring'''




            __snake_case   ,      __snake_case         =					self.prepare_init_args_and_inputs_for_common()

            __snake_case         =					16
            __snake_case         =					(16, 32)

            __snake_case         =					self.model_class(**__SCREAMING_SNAKE_CASE					)
            __snake_case         =					model.init(inputs_dict['''prng_key''']      ,					inputs_dict['''sample''']					)
            jax.lax.stop_gradient(__SCREAMING_SNAKE_CASE					)

            __snake_case         =					model.apply(__SCREAMING_SNAKE_CASE      ,					inputs_dict['''sample''']					)

            if isinstance(__SCREAMING_SNAKE_CASE      ,					__SCREAMING_SNAKE_CASE					):
                 __snake_case         =					output.sample

            self.assertIsNotNone(__SCREAMING_SNAKE_CASE					)
            __snake_case         =					inputs_dict['''sample'''].shape
            self.assertEqual(output.shape      ,					__SCREAMING_SNAKE_CASE      ,					'''Input and output shapes do not match'''					)




       def  lowerCAmelCase (				self					) ->					str:



            '''simple docstring'''




            __snake_case         =					'''kwargs''' in inspect.signature(self.model_class.__init__					).parameters
            __snake_case         =					len(self.model_class._deprecated_kwargs					) > 0

            if has_kwarg_in_model_class and not has_deprecated_kwarg:
                 raise ValueError(
                     F'''{self.model_class} has `**kwargs` in its __init__ method but has not defined any deprecated kwargs'''
                     ''' under the `_deprecated_kwargs` class attribute. Make sure to either remove `**kwargs` if there are'''
                     ''' no deprecated arguments or add the deprecated argument with `_deprecated_kwargs ='''
                     ''' [<deprecated_argument>]`'''					)

            if not has_kwarg_in_model_class and has_deprecated_kwarg:
                 raise ValueError(
                     F'''{self.model_class} doesn\'t have `**kwargs` in its __init__ method but has defined deprecated kwargs'''
                     ''' under the `_deprecated_kwargs` class attribute. Make sure to either add the `**kwargs` argument to'''
                     F''' {self.model_class}.__init__ if there are deprecated arguments or remove the deprecated argument'''
                     ''' from `_deprecated_kwargs = [<deprecated_argument>]`'''					)






