






"""simple docstring"""



import importlib
import os
import platform
from pathlib import Path

from packaging import version

from .utils.logging import get_logger


_UpperCamelCase		:  Union[str, Any]						=		get_logger(__name__)

# Datasets
_UpperCamelCase		:  Union[str, Any]						=		'https://s3.amazonaws.com/datasets.huggingface.co/datasets/datasets'
_UpperCamelCase		:  List[Any]						=		'https://cdn-datasets.huggingface.co/datasets/datasets'
_UpperCamelCase		:  Tuple						=		'https://raw.githubusercontent.com/huggingface/datasets/{revision}/datasets/{path}/{name}'

# Metrics
_UpperCamelCase		:  str						=		'https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics'
_UpperCamelCase		:  Optional[Any]						=		'https://cdn-datasets.huggingface.co/datasets/metric'
_UpperCamelCase		:  Dict						=		'https://raw.githubusercontent.com/huggingface/datasets/{revision}/metrics/{path}/{name}'

# Hub
_UpperCamelCase		:  List[str]						=		os.environ.get('HF_ENDPOINT', 'https://huggingface.co')
_UpperCamelCase		:  Optional[Any]						=		HF_ENDPOINT + '/datasets/{repo_id}/resolve/{revision}/{path}'
_UpperCamelCase		:  List[Any]						=		'main'

_UpperCamelCase		:  int						=		version.parse(platform.python_version())

if PY_VERSION < version.parse('3.8'):
    import importlib_metadata
else:
    import importlib.metadata as importlib_metadata

# General environment variables accepted values for booleans
_UpperCamelCase		:  List[str]						=		{'1', 'ON', 'YES', 'TRUE'}
_UpperCamelCase		:  Union[str, Any]						=		ENV_VARS_TRUE_VALUES.union({'AUTO'})


# Imports
_UpperCamelCase		:  Any						=		version.parse(importlib_metadata.version('dill'))
_UpperCamelCase		:  Union[str, Any]						=		version.parse(importlib_metadata.version('pandas'))
_UpperCamelCase		:  Optional[int]						=		version.parse(importlib_metadata.version('pyarrow'))

_UpperCamelCase		:  List[str]						=		os.environ.get('USE_TF', 'AUTO').upper()
_UpperCamelCase		:  str						=		os.environ.get('USE_TORCH', 'AUTO').upper()
_UpperCamelCase		:  Optional[int]						=		os.environ.get('USE_JAX', 'AUTO').upper()

_UpperCamelCase		:  Optional[Any]						=		'N/A'
_UpperCamelCase		:  Optional[Any]						=		False

if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES:
    _UpperCamelCase		:  int						=		importlib.util.find_spec('torch') is not None
    if TORCH_AVAILABLE:
        try:
            _UpperCamelCase		:  List[str]						=		version.parse(importlib_metadata.version('torch'))
            logger.info(F'''PyTorch version {TORCH_VERSION} available.''')
        except importlib_metadata.PackageNotFoundError:
            pass
else:
    logger.info('Disabling PyTorch because USE_TF is set')

_UpperCamelCase		:  Any						=		'N/A'
_UpperCamelCase		:  List[str]						=		False

if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES:
    _UpperCamelCase		:  Dict						=		importlib.util.find_spec('tensorflow') is not None
    if TF_AVAILABLE:
        # For the metadata, we have to look for both tensorflow and tensorflow-cpu
        for package in [
            "tensorflow",
            "tensorflow-cpu",
            "tensorflow-gpu",
            "tf-nightly",
            "tf-nightly-cpu",
            "tf-nightly-gpu",
            "intel-tensorflow",
            "tensorflow-rocm",
            "tensorflow-macos",
        ]:
            try:
                _UpperCamelCase		:  str						=		version.parse(importlib_metadata.version(package))
            except importlib_metadata.PackageNotFoundError:
                continue
            else:
                break
        else:
            _UpperCamelCase		:  str						=		False
    if TF_AVAILABLE:
        if TF_VERSION.major < 2:
            logger.info(F'''TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.''')
            _UpperCamelCase		:  Union[str, Any]						=		False
        else:
            logger.info(F'''TensorFlow version {TF_VERSION} available.''')
else:
    logger.info('Disabling Tensorflow because USE_TORCH is set')


_UpperCamelCase		:  Optional[Any]						=		'N/A'
_UpperCamelCase		:  Dict						=		False

if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES:
    _UpperCamelCase		:  List[Any]						=		importlib.util.find_spec('jax') is not None and importlib.util.find_spec('jaxlib') is not None
    if JAX_AVAILABLE:
        try:
            _UpperCamelCase		:  List[Any]						=		version.parse(importlib_metadata.version('jax'))
            logger.info(F'''JAX version {JAX_VERSION} available.''')
        except importlib_metadata.PackageNotFoundError:
            pass
else:
    logger.info('Disabling JAX because USE_JAX is set to False')


_UpperCamelCase		:  Optional[Any]						=		os.environ.get('USE_BEAM', 'AUTO').upper()
_UpperCamelCase		:  Union[str, Any]						=		'N/A'
_UpperCamelCase		:  Optional[Any]						=		False
if USE_BEAM in ENV_VARS_TRUE_AND_AUTO_VALUES:
    try:
        _UpperCamelCase		:  Optional[int]						=		version.parse(importlib_metadata.version('apache_beam'))
        _UpperCamelCase		:  List[Any]						=		True
        logger.info(F'''Apache Beam version {BEAM_VERSION} available.''')
    except importlib_metadata.PackageNotFoundError:
        pass
else:
    logger.info('Disabling Apache Beam because USE_BEAM is set to False')


# Optional tools for data loading
_UpperCamelCase		:  Any						=		importlib.util.find_spec('sqlalchemy') is not None

# Optional tools for feature decoding
_UpperCamelCase		:  Optional[int]						=		importlib.util.find_spec('PIL') is not None
_UpperCamelCase		:  Union[str, Any]						=		importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.0.31')
_UpperCamelCase		:  Any						=		importlib.util.find_spec('soundfile') is not None and version.parse(
    importlib.import_module('soundfile').__libsndfile_version__
) >= version.parse('1.1.0')

# Optional compression tools
_UpperCamelCase		:  List[Any]						=		importlib.util.find_spec('rarfile') is not None
_UpperCamelCase		:  int						=		importlib.util.find_spec('zstandard') is not None
_UpperCamelCase		:  List[Any]						=		importlib.util.find_spec('lz4') is not None
_UpperCamelCase		:  List[Any]						=		importlib.util.find_spec('py7zr') is not None


# Cache location
_UpperCamelCase		:  Optional[int]						=		'~/.cache'
_UpperCamelCase		:  Optional[int]						=		os.getenv('XDG_CACHE_HOME', DEFAULT_XDG_CACHE_HOME)
_UpperCamelCase		:  int						=		os.path.join(XDG_CACHE_HOME, 'huggingface')
_UpperCamelCase		:  int						=		os.path.expanduser(os.getenv('HF_HOME', DEFAULT_HF_CACHE_HOME))

_UpperCamelCase		:  int						=		os.path.join(HF_CACHE_HOME, 'datasets')
_UpperCamelCase		:  Union[str, Any]						=		Path(os.getenv('HF_DATASETS_CACHE', DEFAULT_HF_DATASETS_CACHE))

_UpperCamelCase		:  Optional[Any]						=		os.path.join(HF_CACHE_HOME, 'metrics')
_UpperCamelCase		:  Optional[Any]						=		Path(os.getenv('HF_METRICS_CACHE', DEFAULT_HF_METRICS_CACHE))

_UpperCamelCase		:  Any						=		os.path.join(HF_CACHE_HOME, 'modules')
_UpperCamelCase		:  Optional[Any]						=		Path(os.getenv('HF_MODULES_CACHE', DEFAULT_HF_MODULES_CACHE))

_UpperCamelCase		:  Optional[Any]						=		'downloads'
_UpperCamelCase		:  Union[str, Any]						=		os.path.join(HF_DATASETS_CACHE, DOWNLOADED_DATASETS_DIR)
_UpperCamelCase		:  int						=		Path(os.getenv('HF_DATASETS_DOWNLOADED_DATASETS_PATH', DEFAULT_DOWNLOADED_DATASETS_PATH))

_UpperCamelCase		:  Optional[Any]						=		'extracted'
_UpperCamelCase		:  List[str]						=		os.path.join(DEFAULT_DOWNLOADED_DATASETS_PATH, EXTRACTED_DATASETS_DIR)
_UpperCamelCase		:  Optional[Any]						=		Path(os.getenv('HF_DATASETS_EXTRACTED_DATASETS_PATH', DEFAULT_EXTRACTED_DATASETS_PATH))

# Download count for the website
_UpperCamelCase		:  str						=		(
    os.environ.get('HF_UPDATE_DOWNLOAD_COUNTS', 'AUTO').upper() in ENV_VARS_TRUE_AND_AUTO_VALUES
)

# Batch size constants. For more info, see:
# https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
_UpperCamelCase		:  Tuple						=		1_0_0_0

# Size of the preloaded record batch in `Dataset.__iter__`
_UpperCamelCase		:  int						=		1_0

# Pickling tables works only for small tables (<4GiB)
# For big tables, we write them on disk instead
_UpperCamelCase		:  List[str]						=		4 << 3_0

# Max shard size in bytes (e.g. to shard parquet datasets in push_to_hub or download_and_prepare)
_UpperCamelCase		:  Optional[Any]						=		'500MB'

# Parquet configuration
_UpperCamelCase		:  Tuple						=		1_0_0
_UpperCamelCase		:  Union[str, Any]						=		1_0_0
_UpperCamelCase		:  Optional[Any]						=		1_0_0

# Offline mode
_UpperCamelCase		:  List[str]						=		os.environ.get('HF_DATASETS_OFFLINE', 'AUTO').upper() in ENV_VARS_TRUE_VALUES

# In-memory
_UpperCamelCase		:  Dict						=		0  # Disabled
_UpperCamelCase		:  Union[str, Any]						=		float(os.environ.get('HF_DATASETS_IN_MEMORY_MAX_SIZE', DEFAULT_IN_MEMORY_MAX_SIZE))

# File names
_UpperCamelCase		:  Optional[int]						=		'dataset.arrow'
_UpperCamelCase		:  Dict						=		'indices.arrow'
_UpperCamelCase		:  str						=		'state.json'
_UpperCamelCase		:  str						=		'dataset_info.json'
_UpperCamelCase		:  List[str]						=		'dataset_infos.json'
_UpperCamelCase		:  Any						=		'LICENSE'
_UpperCamelCase		:  Tuple						=		'metric_info.json'
_UpperCamelCase		:  Union[str, Any]						=		'dataset_dict.json'

_UpperCamelCase		:  Optional[int]						=		'datasets_modules'

_UpperCamelCase		:  str						=		2_5_5

# Streaming
_UpperCamelCase		:  List[str]						=		2_0
_UpperCamelCase		:  Tuple						=		5

# Datasets without script
_UpperCamelCase		:  Optional[Any]						=		2_0_0
_UpperCamelCase		:  List[str]						=		1_0
_UpperCamelCase		:  Optional[Any]						=		2_0_0

# Progress bars
_UpperCamelCase		:  List[Any]						=		0.0_5  # 20 progress updates per sec










"""simple docstring"""



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation

_UpperCamelCase		:  Union[str, Any]						=		logging.get_logger(__name__)

_UpperCamelCase		:  Tuple						=		{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_UpperCamelCase		:  Any						=		{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

_UpperCamelCase		:  str						=		{
    'gpt2': 1_0_2_4,
    'gpt2-medium': 1_0_2_4,
    'gpt2-large': 1_0_2_4,
    'gpt2-xl': 1_0_2_4,
    'distilgpt2': 1_0_2_4,
}
@lru_cache()
def 						_SCREAMING_SNAKE_CASE			(   ):





  '''simple docstring'''





  lowercase							   =	(
      list(range(ord('!'   )		, ord('~'   ) + 1   )   ) + list(range(ord('¡'   )		, ord('¬'   ) + 1   )   ) + list(range(ord('®'   )		, ord('ÿ'   ) + 1   )   )
  )
  lowercase							   =	bs[:]
  lowercase							   =	0
  for b in range(2**8   ):
    if b not in bs:
      bs.append(__snake_case   )
      cs.append(2**8 + n   )
      n += 1
  lowercase							   =	[chr(__snake_case   ) for n in cs]
  return dict(zip(__snake_case		, __snake_case   )   )
def 						_SCREAMING_SNAKE_CASE			(   __snake_case      :  int   ):





  '''simple docstring'''





  lowercase							   =	set()
  lowercase							   =	word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)   )
    lowercase							   =	char
  return pairs



class 	a					(       a_     ):
    UpperCAmelCase_			:					int     					=VOCAB_FILES_NAMES
    UpperCAmelCase_			:					List[str]     					=PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase_			:					Tuple     					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase_			:					Any     					=["input_ids", "attention_mask"]







    def __init__(      self							,		_lowerCamelCase							,		_lowerCamelCase							,		_lowerCamelCase="replace"							,		_lowerCamelCase="<|endoftext|>"							,		_lowerCamelCase="<|endoftext|>"							,		_lowerCamelCase="<|endoftext|>"							,		_lowerCamelCase=None							,		_lowerCamelCase=False							,		_lowerCamelCase=False							,		**_lowerCamelCase							,		):
      lowercase							   =	AddedToken(_lowerCamelCase							,		lstrip=_lowerCamelCase							,		rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase							,		_lowerCamelCase	) else bos_token
      lowercase							   =	AddedToken(_lowerCamelCase							,		lstrip=_lowerCamelCase							,		rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase							,		_lowerCamelCase	) else eos_token
      lowercase							   =	AddedToken(_lowerCamelCase							,		lstrip=_lowerCamelCase							,		rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase							,		_lowerCamelCase	) else unk_token
      lowercase							   =	AddedToken(_lowerCamelCase							,		lstrip=_lowerCamelCase							,		rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase							,		_lowerCamelCase	) else pad_token
      super().__init__(
          errors=_lowerCamelCase							,		unk_token=_lowerCamelCase							,		bos_token=_lowerCamelCase							,		eos_token=_lowerCamelCase							,		pad_token=_lowerCamelCase							,		add_prefix_space=_lowerCamelCase							,		add_bos_token=_lowerCamelCase							,		**_lowerCamelCase							,		)
      lowercase							   =	add_bos_token

      with open(_lowerCamelCase							,		encoding='utf-8'	) as vocab_handle:
        lowercase							   =	json.load(_lowerCamelCase	)
      lowercase							   =	{v: k for k, v in self.encoder.items()}
      lowercase							   =	errors  # how to handle errors in decoding
      lowercase							   =	bytes_to_unicode()
      lowercase							   =	{v: k for k, v in self.byte_encoder.items()}
      with open(_lowerCamelCase							,		encoding='utf-8'	) as merges_handle:
        lowercase							   =	merges_handle.read().split('\n'	)[1:-1]
      lowercase							   =	[tuple(merge.split()	) for merge in bpe_merges]
      lowercase							   =	dict(zip(_lowerCamelCase							,		range(len(_lowerCamelCase	)	)	)	)
      lowercase							   =	{}
      lowercase							   =	add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      lowercase							   =	re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)







    @property
    def       UpperCamelCase_				(      self	):
      return len(self.encoder	)







    def       UpperCamelCase_				(      self	):
      return dict(self.encoder							,		**self.added_tokens_encoder	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):
      if token in self.cache:
        return self.cache[token]
      lowercase							   =	tuple(_lowerCamelCase	)
      lowercase							   =	get_pairs(_lowerCamelCase	)

      if not pairs:
        return token

      while True:
        lowercase							   =	min(_lowerCamelCase							,		key=lambda _lowerCamelCase	: self.bpe_ranks.get(_lowerCamelCase							,		float('inf'	)	)	)
        if bigram not in self.bpe_ranks:
          break
        lowercase   ,					lowercase							   =	bigram
        lowercase							   =	[]
        lowercase							   =	0
        while i < len(_lowerCamelCase	):
          try:
            lowercase							   =	word.index(_lowerCamelCase							,		_lowerCamelCase	)
          except ValueError:
            new_word.extend(word[i:]	)
            break
          else:
            new_word.extend(word[i:j]	)
            lowercase							   =	j

          if word[i] == first and i < len(_lowerCamelCase	) - 1 and word[i + 1] == second:
            new_word.append(first + second	)
            i += 2
          else:
            new_word.append(word[i]	)
            i += 1
        lowercase							   =	tuple(_lowerCamelCase	)
        lowercase							   =	new_word
        if len(_lowerCamelCase	) == 1:
          break
        else:
          lowercase							   =	get_pairs(_lowerCamelCase	)
      lowercase							   =	' '.join(_lowerCamelCase	)
      lowercase							   =	word
      return word







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase=None	):
      if self.add_bos_token:
        lowercase							   =	[self.bos_token_id]
      else:
        lowercase							   =	[]

      lowercase							   =	bos_token_ids + token_ids_a

      if token_ids_a is None:
        return output

      return output + bos_token_ids + token_ids_a







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase = None							,		_lowerCamelCase = False	):
      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=_lowerCamelCase							,		token_ids_a=_lowerCamelCase							,		already_has_special_tokens=_lowerCamelCase	)

      if not self.add_bos_token:
        return super().get_special_tokens_mask(
            token_ids_a=_lowerCamelCase							,		token_ids_a=_lowerCamelCase							,		already_has_special_tokens=_lowerCamelCase	)

      if token_ids_a is None:
        return [1] + ([0] * len(_lowerCamelCase	))
      return [1] + ([0] * len(_lowerCamelCase	)) + [1] + ([0] * len(_lowerCamelCase	))







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):
      lowercase							   =	[]
      for token in re.findall(self.pat							,		_lowerCamelCase	):
        lowercase							   =	''.join(
            self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase	).split(' '	)	)
      return bpe_tokens







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):
      return self.encoder.get(_lowerCamelCase							,		self.encoder.get(self.unk_token	)	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):
      return self.decoder.get(_lowerCamelCase	)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):
      lowercase							   =	''.join(_lowerCamelCase	)
      lowercase							   =	bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'							,		errors=self.errors	)
      return text







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase = None	):
      if not os.path.isdir(_lowerCamelCase	):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'	)
        return
      lowercase							   =	os.path.join(
          _lowerCamelCase							,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
      lowercase							   =	os.path.join(
          _lowerCamelCase							,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

      with open(_lowerCamelCase							,		'w'							,		encoding='utf-8'	) as f:
        f.write(json.dumps(self.encoder							,		indent=2							,		sort_keys=_lowerCamelCase							,		ensure_ascii=_lowerCamelCase	) + '\n'	)

      lowercase							   =	0
      with open(_lowerCamelCase							,		'w'							,		encoding='utf-8'	) as writer:
        writer.write('#version: 0.2\n'	)
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,		key=lambda _lowerCamelCase	: kv[1]	):
          if index != token_index:
            logger.warning(
                F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                ' Please check that the tokenizer is not corrupted!'	)
            lowercase							   =	token_index
          writer.write(' '.join(_lowerCamelCase	) + '\n'	)
          index += 1

      return vocab_file, merge_file







    def       UpperCamelCase_				(      self							,		_lowerCamelCase							,		_lowerCamelCase=False							,		**_lowerCamelCase	):
      lowercase							   =	kwargs.pop('add_prefix_space'							,		self.add_prefix_space	)
      if is_split_into_words or add_prefix_space:
        lowercase							   =	' ' + text
      return (text, kwargs)







    def       UpperCamelCase_				(      self							,		_lowerCamelCase	):
      lowercase							   =	[]
      for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(_lowerCamelCase							,		add_special_tokens=_lowerCamelCase	) + [self.eos_token_id]	)
      if len(_lowerCamelCase	) > self.model_max_length:
        lowercase							   =	input_ids[-self.model_max_length :]
      return input_ids


