




import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      _UpperCamelCase (			lowerCAmelCase		,    unittest.TestCase						):
      UpperCAmelCase_            =						HerbertTokenizer
      UpperCAmelCase_            =						HerbertTokenizerFast
      UpperCAmelCase_            =						True
      def 		UpperCAmelCase_  (     self				:str )       ->     List[Any]:
       super().setUp()

       # Use a simpler test file without japanese/chinese characters
       with open(f'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''	, encoding="utf-8" ) as f_data:
        UpperCAmelCase__								=  f_data.read().replace("\n\n"	, "\n" ).strip()

       UpperCAmelCase__								=  [
           "<s>",
           "</s>",
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "w</w>",
           "r</w>",
           "t</w>",
           "lo",
           "low",
           "er</w>",
           "low</w>",
           "lowest</w>",
           "newer</w>",
           "wider</w>",
           ",</w>",
           "<unk>",
       ]
       UpperCAmelCase__								=  dict(zip(lowerCamelCase	, range(len(lowerCamelCase ) ) ) )
       UpperCAmelCase__								=  ["l o 123", "lo w 1456", "e r</w> 1789", ""]

       UpperCAmelCase__								=  os.path.join(self.tmpdirname	, VOCAB_FILES_NAMES["vocab_file"] )
       UpperCAmelCase__								=  os.path.join(self.tmpdirname	, VOCAB_FILES_NAMES["merges_file"] )
       with open(self.vocab_file	, "w" ) as fp:
        fp.write(json.dumps(lowerCamelCase ) )
       with open(self.merges_file	, "w" ) as fp:
        fp.write("\n".join(lowerCamelCase ) )
      def 		UpperCAmelCase_  (     self				:List[Any]	, lowerCamelCase				:str )       ->     Dict:
       UpperCAmelCase__								=  "lower newer"
       UpperCAmelCase__								=  "lower newer"
       return input_text, output_text
      def 		UpperCAmelCase_  (     self				:int )       ->     List[str]:
       UpperCAmelCase__								=  self.tokenizer_class(vocab_file=self.vocab_file	, merges_file=self.merges_file )

       UpperCAmelCase__								=  "lower"
       UpperCAmelCase__								=  ["low", "er</w>"]
       UpperCAmelCase__								=  tokenizer.tokenize(lowerCamelCase )
       self.assertListEqual(lowerCamelCase	, lowerCamelCase )

       UpperCAmelCase__								=  tokens + ["<unk>"]
       UpperCAmelCase__								=  [16, 17, 23]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase )	, lowerCamelCase )
      def 		UpperCAmelCase_  (     self				:Optional[Any] )       ->     Optional[Any]:
       if not self.test_rust_tokenizer:
        return

       UpperCAmelCase__								=  self.get_tokenizer()
       UpperCAmelCase__								=  self.get_rust_tokenizer()

       UpperCAmelCase__								=  "lower,newer"

       UpperCAmelCase__								=  tokenizer.tokenize(lowerCamelCase )
       UpperCAmelCase__								=  rust_tokenizer.tokenize(lowerCamelCase )
       self.assertListEqual(lowerCamelCase	, lowerCamelCase )

       UpperCAmelCase__								=  tokenizer.encode(lowerCamelCase	, add_special_tokens=lowerCamelCase )
       UpperCAmelCase__								=  rust_tokenizer.encode(lowerCamelCase	, add_special_tokens=lowerCamelCase )
       self.assertListEqual(lowerCamelCase	, lowerCamelCase )

       UpperCAmelCase__								=  self.get_rust_tokenizer()
       UpperCAmelCase__								=  tokenizer.encode(lowerCamelCase )
       UpperCAmelCase__								=  rust_tokenizer.encode(lowerCamelCase )
       self.assertListEqual(lowerCamelCase	, lowerCamelCase )







      @slow
      def 		UpperCAmelCase_  (     self				:Any )       ->     Optional[Any]:
       UpperCAmelCase__								=  self.tokenizer_class.from_pretrained("allegro/herbert-base-cased" )

       UpperCAmelCase__								=  tokenizer.encode("konstruowanie sekwencji"	, add_special_tokens=lowerCamelCase )
       UpperCAmelCase__								=  tokenizer.encode("konstruowanie wielu sekwencji"	, add_special_tokens=lowerCamelCase )

       UpperCAmelCase__								=  tokenizer.build_inputs_with_special_tokens(lowerCamelCase )
       UpperCAmelCase__								=  tokenizer.build_inputs_with_special_tokens(lowerCamelCase	, lowerCamelCase )

       assert encoded_sentence == [0] + text + [2]
       assert encoded_pair == [0] + text + [2] + text_a + [2]











from __future__ import annotations



def   lowerCAmelCase						(						_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				int		):




 """simple docstring"""







 assert (
     c % greatest_common_divisor(_lowerCAmelCase  ,							_lowerCAmelCase		) == 0
 )  # greatest_common_divisor(a,b) function implemented below
 ((UpperCAmelCase__)	,	(UpperCAmelCase__)	,	(UpperCAmelCase__))								=  extended_gcd(_lowerCAmelCase  ,							_lowerCAmelCase		)  # extended_gcd(a,b) function implemented below
 UpperCAmelCase__								=  c / d
 return (r * x, r * y)



def   lowerCAmelCase						(						_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				int = 2		):




 """simple docstring"""







 ((UpperCAmelCase__)	,	(UpperCAmelCase__))								=  diophantine(_lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase		)  # Initial value
 UpperCAmelCase__								=  greatest_common_divisor(_lowerCAmelCase  ,							_lowerCAmelCase		)
 UpperCAmelCase__								=  a // d
 UpperCAmelCase__								=  b // d

 for i in range(_lowerCAmelCase		):
  UpperCAmelCase__								=  xa + i * q
  UpperCAmelCase__								=  ya - i * p
  print(_lowerCAmelCase  ,							_lowerCAmelCase		)



def   lowerCAmelCase						(						_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				int		):




 """simple docstring"""







 if a < b:
  UpperCAmelCase__	,	UpperCAmelCase__								=  b, a

 while a % b != 0:
  UpperCAmelCase__	,	UpperCAmelCase__								=  b, a % b

 return b



def   lowerCAmelCase						(						_lowerCAmelCase							:				int  ,							_lowerCAmelCase							:				int		):




 """simple docstring"""







 assert a >= 0 and b >= 0

 if b == 0:
  UpperCAmelCase__	,	UpperCAmelCase__	,	UpperCAmelCase__								=  a, 1, 0
 else:
  ((UpperCAmelCase__)	,	(UpperCAmelCase__)	,	(UpperCAmelCase__))								=  extended_gcd(_lowerCAmelCase  ,							a % b		)
  UpperCAmelCase__								=  q
  UpperCAmelCase__								=  p - q * (a // b)

 assert a % d == 0 and b % d == 0
 assert d == a * x + b * y

 return (d, x, y)


if __name__ == "__main__":
  from doctest import testmod

  testmod(name="diophantine", verbose=True)
  testmod(name="diophantine_all_soln", verbose=True)
  testmod(name="extended_gcd", verbose=True)
  testmod(name="greatest_common_divisor", verbose=True)





