"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

UpperCamelCase_    :    Optional[Any]											=						logging.get_logger(__name__)

UpperCamelCase_    :    Union[str, Any]											=						{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

UpperCamelCase_    :    Dict											=						{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
}

UpperCamelCase_    :    Union[str, Any]											=						{
    "gpt2": 1024,
    "gpt2-medium": 1024,
    "gpt2-large": 1024,
    "gpt2-xl": 1024,
    "distilgpt2": 1024,
}


@lru_cache()
def 					a__ (						)->		Optional[Any]:
						UpperCAmelCase__   		=						(
						    list(range(ord("""!"""							)						,     ord("""~"""							) + 1							)							) + list(range(ord("""¡"""							)						,     ord("""¬"""							) + 1							)							) + list(range(ord("""®"""							)						,     ord("""ÿ"""							) + 1							)							)
						)
						UpperCAmelCase__   		=						bs[:]
						UpperCAmelCase__   		=						0
						for b in range(2**8							):
												if b not in bs:
																		bs.append(UpperCAmelCase__							)
																		cs.append(2**8 + n							)
																		n += 1
						UpperCAmelCase__   		=						[chr(UpperCAmelCase__							) for n in cs]
						return dict(zip(UpperCAmelCase__						,     UpperCAmelCase__							)							)


def 					a__ (						UpperCAmelCase__    :     Optional[int]							)->		Optional[Any]:
						UpperCAmelCase__   		=						set()
						UpperCAmelCase__   		=						word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)							)
												UpperCAmelCase__   		=						char
						return pairs







class      _lowercase						(				SCREAMING_SNAKE_CASE_					):
				lowerCamelCase__									=    VOCAB_FILES_NAMES
				lowerCamelCase__									=    PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase__									=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase__									=    ['input_ids', 'attention_mask']

				def __init__(       self    :      Dict    ,      _lowercase    :      Dict    ,      _lowercase    :      Optional[int]    ,      _lowercase    :      str="replace"    ,      _lowercase    :      Optional[int]="<|endoftext|>"    ,      _lowercase    :      Optional[Any]="<|endoftext|>"    ,      _lowercase    :      Union[str, Any]="<|endoftext|>"    ,      _lowercase    :      str=None    ,      _lowercase    :      Dict=False    ,      _lowercase    :      List[Any]=False    ,      **_lowercase    :      str    ,      ):


										'''simple docstring'''
										UpperCAmelCase__   		=						AddedToken(SCREAMING_SNAKE_CASE_    ,      lstrip=SCREAMING_SNAKE_CASE_    ,      rstrip=SCREAMING_SNAKE_CASE_       ) if isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_       ) else bos_token
										UpperCAmelCase__   		=						AddedToken(SCREAMING_SNAKE_CASE_    ,      lstrip=SCREAMING_SNAKE_CASE_    ,      rstrip=SCREAMING_SNAKE_CASE_       ) if isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_       ) else eos_token
										UpperCAmelCase__   		=						AddedToken(SCREAMING_SNAKE_CASE_    ,      lstrip=SCREAMING_SNAKE_CASE_    ,      rstrip=SCREAMING_SNAKE_CASE_       ) if isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_       ) else unk_token
										UpperCAmelCase__   		=						AddedToken(SCREAMING_SNAKE_CASE_    ,      lstrip=SCREAMING_SNAKE_CASE_    ,      rstrip=SCREAMING_SNAKE_CASE_       ) if isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_       ) else pad_token
										super().__init__(
										    errors=SCREAMING_SNAKE_CASE_    ,      unk_token=SCREAMING_SNAKE_CASE_    ,      bos_token=SCREAMING_SNAKE_CASE_    ,      eos_token=SCREAMING_SNAKE_CASE_    ,      pad_token=SCREAMING_SNAKE_CASE_    ,      add_prefix_space=SCREAMING_SNAKE_CASE_    ,      add_bos_token=SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_    ,      )
										UpperCAmelCase__   		=						add_bos_token

										with open(SCREAMING_SNAKE_CASE_    ,      encoding="""utf-8"""       ) as vocab_handle:
																UpperCAmelCase__   		=						json.load(SCREAMING_SNAKE_CASE_       )
										UpperCAmelCase__   		=						{v: k for k, v in self.encoder.items()}
										UpperCAmelCase__   		=						errors  # how to handle errors in decoding
										UpperCAmelCase__   		=						bytes_to_unicode()
										UpperCAmelCase__   		=						{v: k for k, v in self.byte_encoder.items()}
										with open(SCREAMING_SNAKE_CASE_    ,      encoding="""utf-8"""       ) as merges_handle:
																UpperCAmelCase__   		=						merges_handle.read().split("""\n"""       )[1:-1]
										UpperCAmelCase__   		=						[tuple(merge.split()       ) for merge in bpe_merges]
										UpperCAmelCase__   		=						dict(zip(SCREAMING_SNAKE_CASE_    ,      range(len(SCREAMING_SNAKE_CASE_       )       )       )       )
										UpperCAmelCase__   		=						{}
										UpperCAmelCase__   		=						add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										UpperCAmelCase__   		=						re.compile(r"""\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""       )

				@property
				def     A	(       self    :      List[Any]       ):


										'''simple docstring'''
										return len(self.encoder       )

				def     A	(       self    :      Optional[int]       ):


										'''simple docstring'''
										return dict(self.encoder    ,      **self.added_tokens_encoder       )

				def     A	(       self    :      Dict    ,      _lowercase    :      Any       ):


										'''simple docstring'''






										if token in self.cache:
																return self.cache[token]
										UpperCAmelCase__   		=						tuple(SCREAMING_SNAKE_CASE_       )
										UpperCAmelCase__   		=						get_pairs(SCREAMING_SNAKE_CASE_       )

										if not pairs:
																return token

										while True:
																UpperCAmelCase__   		=						min(SCREAMING_SNAKE_CASE_    ,      key=lambda _lowercase       : self.bpe_ranks.get(SCREAMING_SNAKE_CASE_    ,      float("""inf"""       )       )       )
																if bigram not in self.bpe_ranks:
																						break
																UpperCAmelCase__		, UpperCAmelCase__   		=						bigram
																UpperCAmelCase__   		=						[]
																UpperCAmelCase__   		=						0
																while i < len(SCREAMING_SNAKE_CASE_       ):
																						try:
																												UpperCAmelCase__   		=						word.index(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_       )
																						except ValueError:
																												new_word.extend(word[i:]       )
																												break
																						else:
																												new_word.extend(word[i:j]       )
																												UpperCAmelCase__   		=						j

																						if word[i] == first and i < len(SCREAMING_SNAKE_CASE_       ) - 1 and word[i + 1] == second:
																												new_word.append(first + second       )
																												i += 2
																						else:
																												new_word.append(word[i]       )
																												i += 1
																UpperCAmelCase__   		=						tuple(SCREAMING_SNAKE_CASE_       )
																UpperCAmelCase__   		=						new_word
																if len(SCREAMING_SNAKE_CASE_       ) == 1:
																						break
																else:
																						UpperCAmelCase__   		=						get_pairs(SCREAMING_SNAKE_CASE_       )
										UpperCAmelCase__   		=						""" """.join(SCREAMING_SNAKE_CASE_       )
										UpperCAmelCase__   		=						word
										return word

				def     A	(       self    :      List[str]    ,      _lowercase    :      Union[str, Any]    ,      _lowercase    :      Optional[Any]=None       ):


										'''simple docstring'''






										if self.add_bos_token:
																UpperCAmelCase__   		=						[self.bos_token_id]
										else:
																UpperCAmelCase__   		=						[]

										UpperCAmelCase__   		=						bos_token_ids + token_ids_a

										if token_ids_a is None:
																return output

										return output + bos_token_ids + token_ids_a

				def     A	(       self    :      Tuple    ,      _lowercase    :      Tuple    ,      _lowercase    :      Dict = None    ,      _lowercase    :      Union[str, Any] = False       ):


										'''simple docstring'''






										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=SCREAMING_SNAKE_CASE_    ,      token_ids_a=SCREAMING_SNAKE_CASE_    ,      already_has_special_tokens=SCREAMING_SNAKE_CASE_       )

										if not self.add_bos_token:
																return super().get_special_tokens_mask(
																    token_ids_a=SCREAMING_SNAKE_CASE_    ,      token_ids_a=SCREAMING_SNAKE_CASE_    ,      already_has_special_tokens=SCREAMING_SNAKE_CASE_       )

										if token_ids_a is None:
																return [1] + ([0] * len(SCREAMING_SNAKE_CASE_       ))
										return [1] + ([0] * len(SCREAMING_SNAKE_CASE_       )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE_       ))

				def     A	(       self    :      str    ,      _lowercase    :      Union[str, Any]       ):


										'''simple docstring'''
										UpperCAmelCase__   		=						[]
										for token in re.findall(self.pat    ,      SCREAMING_SNAKE_CASE_       ):
																UpperCAmelCase__   		=						"""""".join(
																    self.byte_encoder[b] for b in token.encode("""utf-8"""       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(SCREAMING_SNAKE_CASE_       ).split(""" """       )       )
										return bpe_tokens

				def     A	(       self    :      str    ,      _lowercase    :      Tuple       ):


										'''simple docstring'''
										return self.encoder.get(SCREAMING_SNAKE_CASE_    ,      self.encoder.get(self.unk_token       )       )

				def     A	(       self    :      Tuple    ,      _lowercase    :      str       ):


										'''simple docstring'''
										return self.decoder.get(SCREAMING_SNAKE_CASE_       )

				def     A	(       self    :      Dict    ,      _lowercase    :      Any       ):


										'''simple docstring'''
										UpperCAmelCase__   		=						"""""".join(SCREAMING_SNAKE_CASE_       )
										UpperCAmelCase__   		=						bytearray([self.byte_decoder[c] for c in text]       ).decode("""utf-8"""    ,      errors=self.errors       )
										return text

				def     A	(       self    :      List[Any]    ,      _lowercase    :      Union[str, Any]    ,      _lowercase    :      List[str] = None       ):


										'''simple docstring'''






										if not os.path.isdir(SCREAMING_SNAKE_CASE_       ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory'       )
																return
										UpperCAmelCase__   		=						os.path.join(
										    SCREAMING_SNAKE_CASE_    ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]       )
										UpperCAmelCase__   		=						os.path.join(
										    SCREAMING_SNAKE_CASE_    ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]       )

										with open(SCREAMING_SNAKE_CASE_    ,      """w"""    ,      encoding="""utf-8"""       ) as f:
																f.write(json.dumps(self.encoder    ,      indent=2    ,      sort_keys=SCREAMING_SNAKE_CASE_    ,      ensure_ascii=SCREAMING_SNAKE_CASE_       ) + """\n"""       )

										UpperCAmelCase__   		=						0
										with open(SCREAMING_SNAKE_CASE_    ,      """w"""    ,      encoding="""utf-8"""       ) as writer:
																writer.write("""#version: 0.2\n"""       )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,      key=lambda _lowercase       : kv[1]       ):
																						if index != token_index:
																												logger.warning(
																												    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																												    """ Please check that the tokenizer is not corrupted!"""       )
																												UpperCAmelCase__   		=						token_index
																						writer.write(""" """.join(SCREAMING_SNAKE_CASE_       ) + """\n"""       )
																						index += 1

										return vocab_file, merge_file

				def     A	(       self    :      List[str]    ,      _lowercase    :      List[str]    ,      _lowercase    :      Any=False    ,      **_lowercase    :      List[str]       ):


										'''simple docstring'''
										UpperCAmelCase__   		=						kwargs.pop("""add_prefix_space"""    ,      self.add_prefix_space       )
										if is_split_into_words or add_prefix_space:
																UpperCAmelCase__   		=						""" """ + text
										return (text, kwargs)

				def     A	(       self    :      Optional[int]    ,      _lowercase    :      List[Any]       ):


										'''simple docstring'''
										UpperCAmelCase__   		=						[]
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(SCREAMING_SNAKE_CASE_    ,      add_special_tokens=SCREAMING_SNAKE_CASE_       ) + [self.eos_token_id]       )
										if len(SCREAMING_SNAKE_CASE_       ) > self.model_max_length:
																UpperCAmelCase__   		=						input_ids[-self.model_max_length :]
										return input_ids


"""simple docstring"""







from collections import namedtuple

UpperCamelCase_    :    Any											=						namedtuple("""from_to""", """from_ to""")

UpperCamelCase_    :    Union[str, Any]											=						{
    """millimeter""": """mm""",
    """centimeter""": """cm""",
    """meter""": """m""",
    """kilometer""": """km""",
    """inch""": """in""",
    """inche""": """in""",  # Trailing 's' has been stripped off
    """feet""": """ft""",
    """foot""": """ft""",
    """yard""": """yd""",
    """mile""": """mi""",
}

UpperCamelCase_    :    str											=						{
    """mm""": from_to(0.001, 1000),
    """cm""": from_to(0.01, 100),
    """m""": from_to(1, 1),
    """km""": from_to(1000, 0.001),
    """in""": from_to(0.0254, 39.3701),
    """ft""": from_to(0.3048, 3.2_8084),
    """yd""": from_to(0.9144, 1.0_9361),
    """mi""": from_to(1609.34, 0.0_0062_1371),
}


def 					a__ (						UpperCAmelCase__    :     float						,     UpperCAmelCase__    :     str						,     UpperCAmelCase__    :     str     )->		float:
						UpperCAmelCase__   		=						from_type.lower().rstrip("""s"""     )
						UpperCAmelCase__   		=						TYPE_CONVERSION.get(UpperCAmelCase__						,     UpperCAmelCase__     )
						UpperCAmelCase__   		=						to_type.lower().rstrip("""s"""     )
						UpperCAmelCase__   		=						TYPE_CONVERSION.get(UpperCAmelCase__						,     UpperCAmelCase__     )
						if new_from not in METRIC_CONVERSION:
												UpperCAmelCase__   		=						(
												    f'Invalid \'from_type\' value: {from_type!r}.\n'
												    f'Conversion abbreviations are: {", ".join(UpperCAmelCase__     )}'
												)
												raise ValueError(UpperCAmelCase__     )
						if new_to not in METRIC_CONVERSION:
												UpperCAmelCase__   		=						(
												    f'Invalid \'to_type\' value: {to_type!r}.\n'
												    f'Conversion abbreviations are: {", ".join(UpperCAmelCase__     )}'
												)
												raise ValueError(UpperCAmelCase__     )
						return value * METRIC_CONVERSION[new_from].from_ * METRIC_CONVERSION[new_to].to


if __name__ == "__main__":
						import doctest

						doctest.testmod()

