
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


a_								=   logging.get_logger(__name__)

a_								=   {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_								=   {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
    """tokenizer_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"""},
}

a_								=   {
    """openai-gpt""": 512,
}



class 	__lowerCAmelCase       (		lowerCAmelCase__				):
 lowerCAmelCase__       =					VOCAB_FILES_NAMES
 lowerCAmelCase__       =					PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       =					["""input_ids""", """attention_mask"""]
 lowerCAmelCase__       =					OpenAIGPTTokenizer




 def __init__(  self		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase="<unk>"		,			**__UpperCAmelCase			):



    '''simple docstring'''
    super().__init__(__UpperCAmelCase		,			__UpperCAmelCase		,			tokenizer_file=__UpperCAmelCase		,			unk_token=__UpperCAmelCase		,			**__UpperCAmelCase			)




 @property
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    return True




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase = None			):



    '''simple docstring'''
    __lowerCamelCase  							=      self._tokenizer.model.save(__UpperCAmelCase		,			name=__UpperCAmelCase			)
    return tuple(__UpperCAmelCase			)











import platform
from argparse import ArgumentParser

import huggingface_hub
import pandas
import pyarrow

from datasets import __version__ as version
from datasets.commands import BaseDatasetsCLICommand





def 						a__     (						_UpperCamelCase     :	Optional[Any]							):
   return EnvironmentCommand()





class 	__lowerCAmelCase       (		lowerCAmelCase__				):




 @staticmethod
 def     lowerCamelCase (  __UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      parser.add_parser('''env'''		,			help='''Print relevant system environment info.'''			)
    download_parser.set_defaults(func=__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      {
        '''`datasets` version''': version,
        '''Platform''': platform.platform(),
        '''Python version''': platform.python_version(),
        '''Huggingface_hub version''': huggingface_hub.__version__,
        '''PyArrow version''': pyarrow.__version__,
        '''Pandas version''': pandas.__version__,
    }

    print('''\nCopy-and-paste the text below in your GitHub issue.\n'''			)
    print(self.format_dict(__UpperCAmelCase			)			)

    return info




 @staticmethod
 def     lowerCamelCase (  __UpperCAmelCase			):



    '''simple docstring'''
    return "\n".join([F"""- {prop}: {val}""" for prop, val in d.items()]			) + "\n"






