




'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation

A			   =logging.get_logger(__name__)

A			   ={
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

A			   ={
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

A			   ={
    'gpt2': 10_24,
    'gpt2-medium': 10_24,
    'gpt2-large': 10_24,
    'gpt2-xl': 10_24,
    'distilgpt2': 10_24,
}





@lru_cache()
def 						snake_case_							():
    UpperCAmelCase      	=		(
        list(range(ord('''!'''					)   ,       ord('''~'''					) + 1					)					) + list(range(ord('''¡'''					)   ,       ord('''¬'''					) + 1					)					) + list(range(ord('''®'''					)   ,       ord('''ÿ'''					) + 1					)					)
    )
    UpperCAmelCase      	=		bs[:]
    UpperCAmelCase      	=		0
    for b in range(2**8					):
        if b not in bs:
            bs.append(_a					)
            cs.append(2**8 + n					)
            n += 1
    UpperCAmelCase      	=		[chr(_a					) for n in cs]
    return dict(zip(_a   ,       _a					)					)





def 						snake_case_							(_a     :							Dict					):
    UpperCAmelCase      	=		set()
    UpperCAmelCase      	=		word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)					)
        UpperCAmelCase      	=		char
    return pairs







class       _a  (     __a			):
       __a    :					Any           = VOCAB_FILES_NAMES
       __a    :					str           = PRETRAINED_VOCAB_FILES_MAP
       __a    :					int           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __a    :					Union[str, Any]           = ["""input_ids""", """attention_mask"""]
       def __init__(     self						:     Any       ,    lowercase						:     List[str]       ,    lowercase						:     int       ,    lowercase						:     Dict="replace"       ,    lowercase						:     Tuple="<|endoftext|>"       ,    lowercase						:     Optional[Any]="<|endoftext|>"       ,    lowercase						:     List[str]="<|endoftext|>"       ,    lowercase						:     Tuple=None       ,    lowercase						:     Dict=False       ,    lowercase						:     Optional[int]=False       ,    **lowercase						:     int       ,    ):



           '''simple docstring'''

           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else bos_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else eos_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else unk_token
           UpperCAmelCase      	=		AddedToken(lowercase       ,    lstrip=lowercase       ,    rstrip=lowercase    ) if isinstance(lowercase       ,    lowercase    ) else pad_token
           super().__init__(
               errors=lowercase       ,    unk_token=lowercase       ,    bos_token=lowercase       ,    eos_token=lowercase       ,    pad_token=lowercase       ,    add_prefix_space=lowercase       ,    add_bos_token=lowercase       ,    **lowercase       ,    )
           UpperCAmelCase      	=		add_bos_token

           with open(lowercase       ,    encoding='''utf-8'''    ) as vocab_handle:
               UpperCAmelCase      	=		json.load(lowercase    )
           UpperCAmelCase      	=		{v: k for k, v in self.encoder.items()}
           UpperCAmelCase      	=		errors  # how to handle errors in decoding
           UpperCAmelCase      	=		bytes_to_unicode()
           UpperCAmelCase      	=		{v: k for k, v in self.byte_encoder.items()}
           with open(lowercase       ,    encoding='''utf-8'''    ) as merges_handle:
               UpperCAmelCase      	=		merges_handle.read().split('''\n'''    )[1:-1]
           UpperCAmelCase      	=		[tuple(merge.split()    ) for merge in bpe_merges]
           UpperCAmelCase      	=		dict(zip(lowercase       ,    range(len(lowercase    )    )    )    )
           UpperCAmelCase      	=		{}
           UpperCAmelCase      	=		add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           UpperCAmelCase      	=		re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )
       @property
       def    A				(     self						:     List[Any]    ):



           '''simple docstring'''

           return len(self.encoder    )
       def    A				(     self						:     List[str]    ):



           '''simple docstring'''

           return dict(self.encoder       ,    **self.added_tokens_encoder    )
       def    A				(     self						:     Tuple       ,    lowercase						:     Any    ):



           '''simple docstring'''

           if token in self.cache:
               return self.cache[token]
           UpperCAmelCase      	=		tuple(lowercase    )
           UpperCAmelCase      	=		get_pairs(lowercase    )

           if not pairs:
               return token

           while True:
               UpperCAmelCase      	=		min(lowercase       ,    key=lambda lowercase    : self.bpe_ranks.get(lowercase       ,    float('''inf'''    )    )    )
               if bigram not in self.bpe_ranks:
                   break
               UpperCAmelCase	,					UpperCAmelCase      	=		bigram
               UpperCAmelCase      	=		[]
               UpperCAmelCase      	=		0
               while i < len(lowercase    ):
                   try:
                       UpperCAmelCase      	=		word.index(lowercase       ,    lowercase    )
                   except ValueError:
                       new_word.extend(word[i:]    )
                       break
                   else:
                       new_word.extend(word[i:j]    )
                       UpperCAmelCase      	=		j

                   if word[i] == first and i < len(lowercase    ) - 1 and word[i + 1] == second:
                       new_word.append(first + second    )
                       i += 2
                   else:
                       new_word.append(word[i]    )
                       i += 1
               UpperCAmelCase      	=		tuple(lowercase    )
               UpperCAmelCase      	=		new_word
               if len(lowercase    ) == 1:
                   break
               else:
                   UpperCAmelCase      	=		get_pairs(lowercase    )
           UpperCAmelCase      	=		''' '''.join(lowercase    )
           UpperCAmelCase      	=		word
           return word
       def    A				(     self						:     Union[str, Any]       ,    lowercase						:     Dict       ,    lowercase						:     Union[str, Any]=None    ):



           '''simple docstring'''

           if self.add_bos_token:
               UpperCAmelCase      	=		[self.bos_token_id]
           else:
               UpperCAmelCase      	=		[]

           UpperCAmelCase      	=		bos_token_ids + token_ids_a

           if token_ids_a is None:
               return output

           return output + bos_token_ids + token_ids_a
       def    A				(     self						:     Any       ,    lowercase						:     List[int]       ,    lowercase						:     Optional[List[int]] = None       ,    lowercase						:     bool = False    ):



           '''simple docstring'''

           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowercase       ,    token_ids_a=lowercase       ,    already_has_special_tokens=lowercase    )

           if not self.add_bos_token:
               return super().get_special_tokens_mask(
                   token_ids_a=lowercase       ,    token_ids_a=lowercase       ,    already_has_special_tokens=lowercase    )

           if token_ids_a is None:
               return [1] + ([0] * len(lowercase    ))
           return [1] + ([0] * len(lowercase    )) + [1] + ([0] * len(lowercase    ))
       def    A				(     self						:     List[Any]       ,    lowercase						:     Optional[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           for token in re.findall(self.pat       ,    lowercase    ):
               UpperCAmelCase      	=		''''''.join(
                   self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase    ).split(''' '''    )    )
           return bpe_tokens
       def    A				(     self						:     str       ,    lowercase						:     Tuple    ):



           '''simple docstring'''

           return self.encoder.get(lowercase       ,    self.encoder.get(self.unk_token    )    )
       def    A				(     self						:     Any       ,    lowercase						:     Dict    ):



           '''simple docstring'''

           return self.decoder.get(lowercase    )
       def    A				(     self						:     Optional[int]       ,    lowercase						:     Optional[int]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		''''''.join(lowercase    )
           UpperCAmelCase      	=		bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''       ,    errors=self.errors    )
           return text
       def    A				(     self						:     str       ,    lowercase						:     str       ,    lowercase						:     Optional[str] = None    ):



           '''simple docstring'''

           if not os.path.isdir(lowercase    ):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
               return
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
           UpperCAmelCase      	=		os.path.join(
               lowercase       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as f:
               f.write(json.dumps(self.encoder       ,    indent=2       ,    sort_keys=lowercase       ,    ensure_ascii=lowercase    ) + '''\n'''    )

           UpperCAmelCase      	=		0
           with open(lowercase       ,    '''w'''       ,    encoding='''utf-8'''    ) as writer:
               writer.write('''#version: 0.2\n'''    )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,    key=lambda lowercase    : kv[1]    ):
                   if index != token_index:
                       logger.warning(
                           f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                           ''' Please check that the tokenizer is not corrupted!'''    )
                       UpperCAmelCase      	=		token_index
                   writer.write(''' '''.join(lowercase    ) + '''\n'''    )
                   index += 1

           return vocab_file, merge_file
       def    A				(     self						:     Tuple       ,    lowercase						:     Tuple       ,    lowercase						:     Any=False       ,    **lowercase						:     Any    ):



           '''simple docstring'''

           UpperCAmelCase      	=		kwargs.pop('''add_prefix_space'''       ,    self.add_prefix_space    )
           if is_split_into_words or add_prefix_space:
               UpperCAmelCase      	=		''' ''' + text
           return (text, kwargs)







       def    A				(     self						:     Dict       ,    lowercase						:     "Conversation"    ):



           '''simple docstring'''

           UpperCAmelCase      	=		[]
           for is_user, text in conversation.iter_texts():
               input_ids.extend(self.encode(lowercase       ,    add_special_tokens=lowercase    ) + [self.eos_token_id]    )
           if len(lowercase    ) > self.model_max_length:
               UpperCAmelCase      	=		input_ids[-self.model_max_length :]
           return input_ids










'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class       _a  (     __a			):
       def __init__(     self						:     Any       ,    lowercase						:     NestedDataStructureLike[PathLike]       ,    lowercase						:     Optional[NamedSplit] = None       ,    lowercase						:     Optional[Features] = None       ,    lowercase						:     str = None       ,    lowercase						:     bool = False       ,    lowercase						:     bool = False       ,    lowercase						:     Optional[int] = None       ,    **lowercase						:     str       ,    ):



           '''simple docstring'''

           super().__init__(
               lowercase       ,    split=lowercase       ,    features=lowercase       ,    cache_dir=lowercase       ,    keep_in_memory=lowercase       ,    streaming=lowercase       ,    num_proc=lowercase       ,    **lowercase       ,    )
           UpperCAmelCase      	=		path_or_paths if isinstance(lowercase       ,    lowercase    ) else {self.split: path_or_paths}
           UpperCAmelCase      	=		Csv(
               cache_dir=lowercase       ,    data_files=lowercase       ,    features=lowercase       ,    **lowercase       ,    )







       def    A				(     self						:     Tuple    ):



           '''simple docstring'''

           if self.streaming:
               UpperCAmelCase      	=		self.builder.as_streaming_dataset(split=self.split    )
           # Build regular (map-style) dataset
           else:
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None
               UpperCAmelCase      	=		None

               self.builder.download_and_prepare(
                   download_config=lowercase       ,    download_mode=lowercase       ,    verification_mode=lowercase       ,    base_path=lowercase       ,    num_proc=self.num_proc       ,    )
               UpperCAmelCase      	=		self.builder.as_dataset(
                   split=self.split       ,    verification_mode=lowercase       ,    in_memory=self.keep_in_memory    )
           return dataset






class       _a  :
       def __init__(     self						:     Optional[int]       ,    lowercase						:     Dataset       ,    lowercase						:     Union[PathLike, BinaryIO]       ,    lowercase						:     Optional[int] = None       ,    lowercase						:     Optional[int] = None       ,    **lowercase						:     int       ,    ):



           '''simple docstring'''

           if num_proc is not None and num_proc <= 0:
               raise ValueError(f"num_proc {num_proc} must be an integer > 0."    )

           UpperCAmelCase      	=		dataset
           UpperCAmelCase      	=		path_or_buf
           UpperCAmelCase      	=		batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
           UpperCAmelCase      	=		num_proc
           UpperCAmelCase      	=		'''utf-8'''
           UpperCAmelCase      	=		to_csv_kwargs
       def    A				(     self						:     Union[str, Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''path_or_buf'''       ,    lowercase    )
           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''header'''       ,    lowercase    )
           UpperCAmelCase      	=		self.to_csv_kwargs.pop('''index'''       ,    lowercase    )

           if isinstance(self.path_or_buf       ,    (str, bytes, os.PathLike)    ):
               with open(self.path_or_buf       ,    '''wb+'''    ) as buffer:
                   UpperCAmelCase      	=		self._write(file_obj=lowercase       ,    header=lowercase       ,    index=lowercase       ,    **self.to_csv_kwargs    )
           else:
               UpperCAmelCase      	=		self._write(file_obj=self.path_or_buf       ,    header=lowercase       ,    index=lowercase       ,    **self.to_csv_kwargs    )
           return written
       def    A				(     self						:     List[str]       ,    lowercase						:     Dict    ):



           '''simple docstring'''

           UpperCAmelCase	,					UpperCAmelCase	,					UpperCAmelCase	,					UpperCAmelCase      	=		args

           UpperCAmelCase      	=		query_table(
               table=self.dataset.data       ,    key=slice(lowercase       ,    offset + self.batch_size    )       ,    indices=self.dataset._indices       ,    )
           UpperCAmelCase      	=		batch.to_pandas().to_csv(
               path_or_buf=lowercase       ,    header=header if (offset == 0) else False       ,    index=lowercase       ,    **lowercase    )
           return csv_str.encode(self.encoding    )







       def    A				(     self						:     List[str]       ,    lowercase						:     BinaryIO       ,    lowercase						:     int       ,    lowercase						:     List[str]       ,    **lowercase						:     List[Any]    ):



           '''simple docstring'''

           UpperCAmelCase      	=		0

           if self.num_proc is None or self.num_proc == 1:
               for offset in logging.tqdm(
                   range(0       ,    len(self.dataset    )       ,    self.batch_size    )       ,    unit='''ba'''       ,    disable=not logging.is_progress_bar_enabled()       ,    desc='''Creating CSV from Arrow format'''       ,    ):
                   UpperCAmelCase      	=		self._batch_csv((offset, header, index, to_csv_kwargs)    )
                   written += file_obj.write(lowercase    )

           else:
               UpperCAmelCase	,					UpperCAmelCase      	=		len(self.dataset    ), self.batch_size
               with multiprocessing.Pool(self.num_proc    ) as pool:
                   for csv_str in logging.tqdm(
                       pool.imap(
                           self._batch_csv       ,    [(offset, header, index, to_csv_kwargs) for offset in range(0       ,    lowercase       ,    lowercase    )]       ,    )       ,    total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,    unit='''ba'''       ,    disable=not logging.is_progress_bar_enabled()       ,    desc='''Creating CSV from Arrow format'''       ,    ):
                       written += file_obj.write(lowercase    )

           return written




