
"""simple docstring"""


import importlib.metadata
import subprocess
import sys






def  _snake_case			(						snake_case__				:       bool = False       ):
	A  				=				False
	if "IPython" in sys.modules:
		A  				=				'google.colab' in str(sys.modules['IPython'].get_ipython()       )

	if in_colab:
		if upgrade:
			A  				=				['pip', 'install', '-U', 'torch']
			subprocess.run(snake_case__		, check=snake_case__       )
		# get the current version of torch
		A  				=				importlib.metadata.version('torch'       )
		A  				=				torch_version[: torch_version.rindex('.'       )]
		A  				=				F'https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-{torch_version_trunc}-cp37-cp37m-linux_x86_64.whl'
		A  				=				['pip', 'install', xla_wheel]
		subprocess.run(snake_case__		, check=snake_case__       )
	else:
		raise RuntimeError('`install_xla` utility works only on google colab.'       )

"""simple docstring"""


import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 			lowerCAmelCase_			( _lowercase						,   unittest.TestCase      ):



						'''simple docstring'''


						_lowerCamelCase:      List[str] =      HerbertTokenizer
						_lowerCamelCase:      Any =      HerbertTokenizerFast
						_lowerCamelCase:      Optional[int] =      True


						def   _SCREAMING_SNAKE_CASE (	self       : Dict )       ->				Tuple:
							super().setUp()

							# Use a simpler test file without japanese/chinese characters
							with open(F'{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'					,encoding='utf-8' ) as f_data:
								A  				=				f_data.read().replace('\n\n'					,'\n' ).strip()

							A  				=				[
							    '<s>',
							    '</s>',
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    'w</w>',
							    'r</w>',
							    't</w>',
							    'lo',
							    'low',
							    'er</w>',
							    'low</w>',
							    'lowest</w>',
							    'newer</w>',
							    'wider</w>',
							    ',</w>',
							    '<unk>',
							]
							A  				=				dict(zip(A_					,range(len(A_ ) ) ) )
							A  				=				['l o 123', 'lo w 1456', 'e r</w> 1789', '']

							A  				=				os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES['vocab_file'] )
							A  				=				os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES['merges_file'] )
							with open(self.vocab_file					,'w' ) as fp:
								fp.write(json.dumps(A_ ) )
							with open(self.merges_file					,'w' ) as fp:
								fp.write('\n'.join(A_ ) )


						def   _SCREAMING_SNAKE_CASE (	self       : Tuple					,A_       : List[str] )       ->				int:
							A  				=				'lower newer'
							A  				=				'lower newer'
							return input_text, output_text


						def   _SCREAMING_SNAKE_CASE (	self       : List[Any] )       ->				int:
							A  				=				self.tokenizer_class(vocab_file=self.vocab_file					,merges_file=self.merges_file )

							A  				=				'lower'
							A  				=				['low', 'er</w>']
							A  				=				tokenizer.tokenize(A_ )
							self.assertListEqual(A_					,A_ )

							A  				=				tokens + ['<unk>']
							A  				=				[16, 17, 23]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(A_ )					,A_ )


						def   _SCREAMING_SNAKE_CASE (	self       : Dict )       ->				Optional[int]:
							if not self.test_rust_tokenizer:
								return

							A  				=				self.get_tokenizer()
							A  				=				self.get_rust_tokenizer()

							A  				=				'lower,newer'

							A  				=				tokenizer.tokenize(A_ )
							A  				=				rust_tokenizer.tokenize(A_ )
							self.assertListEqual(A_					,A_ )

							A  				=				tokenizer.encode(A_					,add_special_tokens=A_ )
							A  				=				rust_tokenizer.encode(A_					,add_special_tokens=A_ )
							self.assertListEqual(A_					,A_ )

							A  				=				self.get_rust_tokenizer()
							A  				=				tokenizer.encode(A_ )
							A  				=				rust_tokenizer.encode(A_ )
							self.assertListEqual(A_					,A_ )




						@slow
						def   _SCREAMING_SNAKE_CASE (	self       : str )       ->				List[Any]:
							A  				=				self.tokenizer_class.from_pretrained('allegro/herbert-base-cased' )

							A  				=				tokenizer.encode('konstruowanie sekwencji'					,add_special_tokens=A_ )
							A  				=				tokenizer.encode('konstruowanie wielu sekwencji'					,add_special_tokens=A_ )

							A  				=				tokenizer.build_inputs_with_special_tokens(A_ )
							A  				=				tokenizer.build_inputs_with_special_tokens(A_					,A_ )

							assert encoded_sentence == [0] + text + [2]
							assert encoded_pair == [0] + text + [2] + text_a + [2]