



import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)



class 	__lowerCAmelCase       :




 def __init__(  self		,			__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      foo




 def __call__(  self			):



    '''simple docstring'''
    return self.foo



class 	__lowerCAmelCase       (		datasets.Dataset				):




 @fingerprint_transform(inplace=__UpperCAmelCase			)
 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):



    '''simple docstring'''
    return DatasetChild(self.data		,			fingerprint=__UpperCAmelCase			)




 @fingerprint_transform(inplace=__UpperCAmelCase			)
 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):



    '''simple docstring'''
    return DatasetChild(self.data		,			fingerprint=__UpperCAmelCase			)



class 	__lowerCAmelCase       :




 def __init__(  self		,			__UpperCAmelCase			):



    '''simple docstring'''
    __lowerCamelCase  							=      callable




 def __call__(  self		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):



    '''simple docstring'''
    if self.callable is not None:
       return self.callable(*__UpperCAmelCase		,			**__UpperCAmelCase			)




 def __getstate__(  self			):



    '''simple docstring'''
    raise pickle.PicklingError()



class 	__lowerCAmelCase       (		lowerCAmelCase__				):




 @require_transformers
 @pytest.mark.integration
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    from transformers import AutoTokenizer

    def encode(__UpperCAmelCase			):
       return tokenizer(__UpperCAmelCase			)

    # TODO: add hash consistency tests across sessions
    __lowerCamelCase  							=      AutoTokenizer.from_pretrained('''bert-base-uncased'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase			: tokenizer(__UpperCAmelCase			)			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      AutoTokenizer.from_pretrained('''bert-base-cased'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase			: tokenizer(__UpperCAmelCase			)			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      AutoTokenizer.from_pretrained('''bert-base-uncased'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase			: tokenizer(__UpperCAmelCase			)			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 @require_transformers
 @pytest.mark.integration
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    from transformers import AutoTokenizer

    __lowerCamelCase  							=      AutoTokenizer.from_pretrained('''gpt2'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    tokenizer('''Hello world !'''			)  # call once to change the tokenizer's cache
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 @require_regex
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    import regex

    __lowerCamelCase  							=      regex.Regex('''foo'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      regex.Regex('''bar'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      regex.Regex('''foo'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)



class 	__lowerCAmelCase       (		lowerCAmelCase__				):




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    def func():
       return foo

    __lowerCamelCase  							=      [0]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      [1]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      [0]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    def func():
       pass

    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()

    def func():
       pass

    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(Foo([1]			)			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(Foo([0]			)			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(Foo([1]			).__call__			)			).hexdigest()
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(Foo([0]			).__call__			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      (
        '''co_argcount''',
        '''co_kwonlyargcount''',
        '''co_nlocals''',
        '''co_stacksize''',
        '''co_flags''',
        '''co_code''',
        '''co_consts''',
        '''co_names''',
        '''co_varnames''',
        '''co_filename''',
        '''co_name''',
        '''co_firstlineno''',
        '''co_lnotab''',
        '''co_freevars''',
        '''co_cellvars''',
    )

    def _create_code_pyaa(*__UpperCAmelCase			):
       if hasattr(args[-3]		,			'''encode'''			):
          __lowerCamelCase  							=      list(__UpperCAmelCase			)
          __lowerCamelCase  							=      args[-3].encode()  # co_lnotab
          __lowerCamelCase  							=      args[-10].encode()  # co_code
       if hasattr(__UpperCAmelCase		,			'''co_posonlyargcount'''			):
          if len(__UpperCAmelCase			) == 16:
             return CodeType(*__UpperCAmelCase			)
          elif len(__UpperCAmelCase			) == 15:
             return CodeType(args[0]		,			0		,			*args[1:]			)
          return CodeType(args[0]		,			0		,			0		,			*args[1:]			)
       elif hasattr(__UpperCAmelCase		,			'''co_kwonlyargcount'''			):
          if len(__UpperCAmelCase			) == 16:
             return CodeType(args[0]		,			*args[2:]			)
          elif len(__UpperCAmelCase			) == 15:
             return CodeType(*__UpperCAmelCase			)
          return CodeType(args[0]		,			0		,			*args[1:]			)
       if len(__UpperCAmelCase			) == 16:
          return CodeType(args[0]		,			*args[3:]			)
       elif len(__UpperCAmelCase			) == 15:
          return CodeType(args[0]		,			*args[2:]			)
       return CodeType(*__UpperCAmelCase			)

    def create_ipython_func(__UpperCAmelCase		,			__UpperCAmelCase			):
       def func():
          return returned_obj

       __lowerCamelCase  							=      func.__code__
       # Use _create_code from dill in order to make it work for different python versions
       if datasets.config.PY_VERSION < version.parse('''3.8'''			):
          __lowerCamelCase  							=      _create_code_pyaa(
              *[getattr(__UpperCAmelCase		,			__UpperCAmelCase			) if k != '''co_filename''' else co_filename for k in code_args_pyaa]			)
       else:
          __lowerCamelCase  							=      code.replace(co_filename=__UpperCAmelCase			)
       return FunctionType(__UpperCAmelCase		,			func.__globals__		,			func.__name__		,			func.__defaults__		,			func.__closure__			)

    __lowerCamelCase							,__lowerCamelCase  							=      '''<ipython-input-2-e0383a102aae>''', [0]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase		,			__UpperCAmelCase			)			)			).hexdigest()
    __lowerCamelCase							,__lowerCamelCase  							=      '''<ipython-input-2-e0383a102aae>''', [1]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase		,			__UpperCAmelCase			)			)			).hexdigest()
    __lowerCamelCase							,__lowerCamelCase  							=      '''<ipython-input-5-713f6613acf3>''', [0]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase		,			__UpperCAmelCase			)			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)

    __lowerCamelCase							,__lowerCamelCase  							=      os.path.join(gettempdir()		,			'''ipykernel_12345'''		,			'''321456789.py'''			), [0]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase		,			__UpperCAmelCase			)			)			).hexdigest()
    __lowerCamelCase							,__lowerCamelCase  							=      os.path.join(gettempdir()		,			'''ipykernel_12345'''		,			'''321456789.py'''			), [1]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase		,			__UpperCAmelCase			)			)			).hexdigest()
    __lowerCamelCase							,__lowerCamelCase  							=      os.path.join(gettempdir()		,			'''ipykernel_12345'''		,			'''654123987.py'''			), [0]
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCAmelCase		,			__UpperCAmelCase			)			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase							,__lowerCamelCase  							=      [0], [1]

    def func():
       return foo, bar

    __lowerCamelCase  							=      '''__main__'''

    def globalvars_mocka_side_effect(__UpperCAmelCase		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):
       return {"foo": foo, "bar": bar}

    def globalvars_mocka_side_effect(__UpperCAmelCase		,			*__UpperCAmelCase		,			**__UpperCAmelCase			):
       return {"bar": bar, "foo": foo}

    with patch('''dill.detect.globalvars'''		,			side_effect=__UpperCAmelCase			) as globalvars_mocka:
       __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
       self.assertGreater(globalvars_mocka.call_count		,			0			)
    with patch('''dill.detect.globalvars'''		,			side_effect=__UpperCAmelCase			) as globalvars_mocka:
       __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
       self.assertGreater(globalvars_mocka.call_count		,			0			)
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)



class 	__lowerCAmelCase       (		lowerCAmelCase__				):




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      Hasher.hash('''hello'''			)
    __lowerCamelCase  							=      Hasher.hash('''hello'''			)
    __lowerCamelCase  							=      Hasher.hash('''there'''			)
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      Hasher.hash(Foo('''hello'''			)			)
    __lowerCamelCase  							=      Hasher.hash(Foo('''hello'''			)			)
    __lowerCamelCase  							=      Hasher.hash(Foo('''there'''			)			)
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      Hasher()
    for x in ["hello", Foo('''hello'''			)]:
       hasher.update(__UpperCAmelCase			)
    __lowerCamelCase  							=      hasher.hexdigest()
    __lowerCamelCase  							=      Hasher()
    for x in ["hello", Foo('''hello'''			)]:
       hasher.update(__UpperCAmelCase			)
    __lowerCamelCase  							=      hasher.hexdigest()
    __lowerCamelCase  							=      Hasher()
    for x in ["there", Foo('''there'''			)]:
       hasher.update(__UpperCAmelCase			)
    __lowerCamelCase  							=      hasher.hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    with self.assertRaises(pickle.PicklingError			):
       Hasher.hash(UnpicklableCallable(Foo('''hello'''			)			)			)




 def     lowerCamelCase (  self			):



    '''simple docstring'''
    __lowerCamelCase  							=      '''abc'''
    __lowerCamelCase  							=      [string, string]  # two strings have the same ids
    __lowerCamelCase  							=      [string, string]
    __lowerCamelCase  							=      json.loads(F"""[\"{string}\", \"{string}\"]"""			)  # two strings have different ids
    self.assertIs(obja[0]		,			__UpperCAmelCase			)
    self.assertIs(obja[0]		,			obja[1]			)
    self.assertIs(obja[0]		,			__UpperCAmelCase			)
    self.assertIs(obja[0]		,			obja[1]			)
    self.assertIsNot(obja[0]		,			__UpperCAmelCase			)
    self.assertIsNot(obja[0]		,			obja[1]			)
    __lowerCamelCase  							=      Hasher.hash(__UpperCAmelCase			)
    __lowerCamelCase  							=      Hasher.hash(__UpperCAmelCase			)
    __lowerCamelCase  							=      Hasher.hash(__UpperCAmelCase			)
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 @require_tiktoken
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    import tiktoken

    __lowerCamelCase  							=      tiktoken.get_encoding('''gpt2'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      tiktoken.get_encoding('''r50k_base'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      tiktoken.get_encoding('''gpt2'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 @require_torch
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    import torch

    __lowerCamelCase  							=      torch.tensor([1.0]			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      torch.tensor([2.0]			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      torch.tensor([1.0]			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)




 @require_spacy
 @require_spacy_model('''en_core_web_sm'''			)
 @require_spacy_model('''fr_core_news_sm'''			)
 @pytest.mark.integration
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    import spacy

    __lowerCamelCase  							=      spacy.load('''en_core_web_sm'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      spacy.load('''fr_core_news_sm'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    __lowerCamelCase  							=      spacy.load('''en_core_web_sm'''			)
    __lowerCamelCase  							=      mda(datasets.utils.py_utils.dumps(__UpperCAmelCase			)			).hexdigest()
    self.assertEqual(__UpperCAmelCase		,			__UpperCAmelCase			)
    self.assertNotEqual(__UpperCAmelCase		,			__UpperCAmelCase			)





@pytest.mark.integration
def 						a__     (						_UpperCamelCase     :	Path							):
   __lowerCamelCase  							=      tmp_path / '''dir1'''
   __lowerCamelCase  							=      tmp_path / '''dir2'''
   dira.mkdir()
   dira.mkdir()
   __lowerCamelCase  							=      '''script.py'''
   __lowerCamelCase  							=      dedent(
       '''
    from datasets.fingerprint import Hasher
    def foo():
        pass
    print(Hasher.hash(foo))
    '''							)
   __lowerCamelCase  							=      dira / script_filename
   __lowerCamelCase  							=      dira / script_filename
   with script_patha.open('''w'''							) as f:
      f.write(_UpperCamelCase							)
   with script_patha.open('''w'''							) as f:
      f.write(_UpperCamelCase							)
   __lowerCamelCase  							=      subprocess.check_output(['''python''', str(_UpperCamelCase							)]							)
   __lowerCamelCase  							=      subprocess.check_output(['''python''', str(_UpperCamelCase							)]							)
   assert fingerprinta == fingerprinta





def 						a__     (						):
   __lowerCamelCase  							=      {'''a''': [0, 1, 2]}
   __lowerCamelCase  							=      DatasetChild(InMemoryTable.from_pydict(_UpperCamelCase							)							)
   __lowerCamelCase  							=      dataset.funca()._fingerprint
   assert expected_fingerprint == dataset.funca()._fingerprint
   assert expected_fingerprint != dataset.funca()._fingerprint

   with Pool(2							) as p:
      assert expected_fingerprint == p.apply_async(dataset.funca							).get()._fingerprint
      assert expected_fingerprint != p.apply_async(dataset.funca							).get()._fingerprint





def 						a__     (						):
   __lowerCamelCase  							=      {'''a''': [0, 1, 2]}



   class 	__lowerCAmelCase       (		datasets.Dataset				):




    @fingerprint_transform(inplace=__UpperCAmelCase			)
    def     lowerCamelCase (  self		,			__UpperCAmelCase			):



       '''simple docstring'''
       return DummyDatasetChild(self.data		,			fingerprint=__UpperCAmelCase			)



   __lowerCamelCase  							=      DummyDatasetChild(InMemoryTable.from_pydict(_UpperCamelCase							)							).func()



   class 	__lowerCAmelCase       (		datasets.Dataset				):




    @fingerprint_transform(inplace=__UpperCAmelCase		,			version='''1.0.0'''			)
    def     lowerCamelCase (  self		,			__UpperCAmelCase			):



       '''simple docstring'''
       return DummyDatasetChild(self.data		,			fingerprint=__UpperCAmelCase			)



   __lowerCamelCase  							=      DummyDatasetChild(InMemoryTable.from_pydict(_UpperCamelCase							)							).func()



   class 	__lowerCAmelCase       (		datasets.Dataset				):




    @fingerprint_transform(inplace=__UpperCAmelCase		,			version='''2.0.0'''			)
    def     lowerCamelCase (  self		,			__UpperCAmelCase			):



       '''simple docstring'''
       return DummyDatasetChild(self.data		,			fingerprint=__UpperCAmelCase			)



   __lowerCamelCase  							=      DummyDatasetChild(InMemoryTable.from_pydict(_UpperCamelCase							)							).func()

   assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}							) == 3





def 						a__     (						):
   # AttributeError: module 'dill._dill' has no attribute 'stack'
   __lowerCamelCase  							=      Hasher()
   hasher.update(lambda _UpperCamelCase							: x							)











from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


a_								=   logging.get_logger(__name__)

a_								=   {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

a_								=   {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
    """tokenizer_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"""},
}

a_								=   {
    """openai-gpt""": 512,
}



class 	__lowerCAmelCase       (		lowerCAmelCase__				):
 lowerCAmelCase__       =					VOCAB_FILES_NAMES
 lowerCAmelCase__       =					PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__       =					["""input_ids""", """attention_mask"""]
 lowerCAmelCase__       =					OpenAIGPTTokenizer




 def __init__(  self		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase=None		,			__UpperCAmelCase="<unk>"		,			**__UpperCAmelCase			):



    '''simple docstring'''
    super().__init__(__UpperCAmelCase		,			__UpperCAmelCase		,			tokenizer_file=__UpperCAmelCase		,			unk_token=__UpperCAmelCase		,			**__UpperCAmelCase			)




 @property
 def     lowerCamelCase (  self			):



    '''simple docstring'''
    return True




 def     lowerCamelCase (  self		,			__UpperCAmelCase		,			__UpperCAmelCase = None			):



    '''simple docstring'''
    __lowerCamelCase  							=      self._tokenizer.model.save(__UpperCAmelCase		,			name=__UpperCAmelCase			)
    return tuple(__UpperCAmelCase			)






