

'''simple docstring'''






import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


snake_case_					  =	threading.Lock()
snake_case_					  =	None

snake_case_					  =	{
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

snake_case_					  =	logging.WARNING

snake_case_					  =	True


def  __lowerCamelCase      (				)							->       List[str]:
      """simple docstring"""

      SCREAMING_SNAKE_CASE_				:      List[str]					=	os.getenv("DIFFUSERS_VERBOSITY"  ,  SCREAMING_SNAKE_CASE_   )
      if env_level_str:
            if env_level_str in log_levels:
                  return log_levels[env_level_str]
            else:
                  logging.getLogger().warning(
                      F"Unknown option DIFFUSERS_VERBOSITY={env_level_str}, "
                      F"has to be one of: { ', '.join(log_levels.keys()   ) }"   )
      return _default_log_level


def  __lowerCamelCase      (				)							->       str:
      """simple docstring"""

      return __name__.split("."   )[0]


def  __lowerCamelCase      (				)							->       logging.Logger:
      """simple docstring"""

      return logging.getLogger(_get_library_name()   )


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      global _default_handler

      with _lock:
            if _default_handler:
                  # This library has already configured the library root logger.
                  return
            SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	logging.StreamHandler()  # Set sys.stderr as stream.
            SCREAMING_SNAKE_CASE_				:      Tuple					=	sys.stderr.flush

            # Apply our default configuration to the library root logger.
            SCREAMING_SNAKE_CASE_				:      Any					=	_get_library_root_logger()
            library_root_logger.addHandler(_default_handler   )
            library_root_logger.setLevel(_get_default_logging_level()   )
            SCREAMING_SNAKE_CASE_				:      str					=	False


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      global _default_handler

      with _lock:
            if not _default_handler:
                  return

            SCREAMING_SNAKE_CASE_				:      int					=	_get_library_root_logger()
            library_root_logger.removeHandler(_default_handler   )
            library_root_logger.setLevel(logging.NOTSET   )
            SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	None


def  __lowerCamelCase      (				)							->       List[Any]:
      """simple docstring"""

      return log_levels


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     Optional[str] = None   )							->       logging.Logger:
      """simple docstring"""

      if name is None:
            SCREAMING_SNAKE_CASE_				:      int					=	_get_library_name()

      _configure_library_root_logger()
      return logging.getLogger(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       int:
      """simple docstring"""

      _configure_library_root_logger()
      return _get_library_root_logger().getEffectiveLevel()


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     int   )							->       None:
      """simple docstring"""

      _configure_library_root_logger()
      _get_library_root_logger().setLevel(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       Any:
      """simple docstring"""

      return set_verbosity(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       List[str]:
      """simple docstring"""

      return set_verbosity(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       Union[str, Any]:
      """simple docstring"""

      return set_verbosity(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       str:
      """simple docstring"""

      return set_verbosity(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      _configure_library_root_logger()

      assert _default_handler is not None
      _get_library_root_logger().removeHandler(_default_handler   )


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      _configure_library_root_logger()

      assert _default_handler is not None
      _get_library_root_logger().addHandler(_default_handler   )


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     logging.Handler   )							->       None:
      """simple docstring"""

      _configure_library_root_logger()

      assert handler is not None
      _get_library_root_logger().addHandler(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				SCREAMING_SNAKE_CASE_					:     logging.Handler   )							->       None:
      """simple docstring"""

      _configure_library_root_logger()

      assert handler is not None and handler not in _get_library_root_logger().handlers
      _get_library_root_logger().removeHandler(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      _configure_library_root_logger()
      SCREAMING_SNAKE_CASE_				:      str					=	False


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      _configure_library_root_logger()
      SCREAMING_SNAKE_CASE_				:      Tuple					=	True


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      SCREAMING_SNAKE_CASE_				:      Tuple					=	_get_library_root_logger().handlers

      for handler in handlers:
            SCREAMING_SNAKE_CASE_				:      Any					=	logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"   )
            handler.setFormatter(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				)							->       None:
      """simple docstring"""

      SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	_get_library_root_logger().handlers

      for handler in handlers:
            handler.setFormatter(SCREAMING_SNAKE_CASE_   )


def  __lowerCamelCase      (				self					:     Any  ,  *SCREAMING_SNAKE_CASE_					:     Dict  ,  **SCREAMING_SNAKE_CASE_					:     Any   )							->       int:
      """simple docstring"""

      SCREAMING_SNAKE_CASE_				:      Optional[int]					=	os.getenv("DIFFUSERS_NO_ADVISORY_WARNINGS"  ,  SCREAMING_SNAKE_CASE_   )
      if no_advisory_warnings:
            return
      self.warning(*SCREAMING_SNAKE_CASE_  ,  **SCREAMING_SNAKE_CASE_   )


snake_case_					  =	warning_advice
class      SCREAMING_SNAKE_CASE__      :




    def __init__( self    ,					*lowercase__    ,					**lowercase__      ):  # pylint: disable=unused-argument






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	args[0] if args else None




    def __iter__( self      ):






          """simple docstring"""






          return iter(self._iterator      )




    def __getattr__( self    ,					lowercase__      ):






          """simple docstring"""






          def empty_fn(*lowercase__    ,					**lowercase__      ):  # pylint: disable=unused-argument
                return

          return empty_fn




    def __enter__( self      ):






          """simple docstring"""






          return self







    def __exit__( self    ,					lowercase__    ,					lowercase__    ,					lowercase__      ):






          """simple docstring"""






          return






class      SCREAMING_SNAKE_CASE__      :




    def __call__( self    ,					*lowercase__    ,					**lowercase__      ):






          """simple docstring"""






          if _tqdm_active:
                return tqdm_lib.tqdm(*lowercase__    ,					**lowercase__      )
          else:
                return EmptyTqdm(*lowercase__    ,					**lowercase__      )




    def   __lowerCamelCase				( self    ,					*lowercase__    ,					**lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      str					=	None
          if _tqdm_active:
                return tqdm_lib.tqdm.set_lock(*lowercase__    ,					**lowercase__      )







    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          if _tqdm_active:
                return tqdm_lib.tqdm.get_lock()






snake_case_					  =	_tqdm_cls()


def  __lowerCamelCase      (				)							->       bool:
      """simple docstring"""

      global _tqdm_active
      return bool(_tqdm_active   )


def  __lowerCamelCase      (				)							->       Optional[Any]:
      """simple docstring"""

      global _tqdm_active
      SCREAMING_SNAKE_CASE_				:      Dict					=	True





def  __lowerCamelCase      (				)							->       List[Any]:
      """simple docstring"""

      global _tqdm_active
      SCREAMING_SNAKE_CASE_				:      int					=	False




'''simple docstring'''






import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      SCREAMING_SNAKE_CASE__      (    _UpperCAmelCase,unittest.TestCase				):
    _A         =							HerbertTokenizer
    _A         =							HerbertTokenizerFast
    _A         =							True




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          super().setUp()

          # Use a simpler test file without japanese/chinese characters
          with open(F"{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"    ,					encoding="utf-8"      ) as f_data:
                SCREAMING_SNAKE_CASE_				:      List[str]					=	f_data.read().replace("\n\n"    ,					"\n"      ).strip()

          SCREAMING_SNAKE_CASE_				:      Dict					=	[
              "<s>",
              "</s>",
              "l",
              "o",
              "w",
              "e",
              "r",
              "s",
              "t",
              "i",
              "d",
              "n",
              "w</w>",
              "r</w>",
              "t</w>",
              "lo",
              "low",
              "er</w>",
              "low</w>",
              "lowest</w>",
              "newer</w>",
              "wider</w>",
              ",</w>",
              "<unk>",
          ]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	dict(zip(lowercase__    ,					range(len(lowercase__      )      )      )      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	["l o 123", "lo w 1456", "e r</w> 1789", ""]

          SCREAMING_SNAKE_CASE_				:      Any					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["vocab_file"]      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES["merges_file"]      )
          with open(self.vocab_file    ,					"w"      ) as fp:
                fp.write(json.dumps(lowercase__      )      )
          with open(self.merges_file    ,					"w"      ) as fp:
                fp.write("\n".join(lowercase__      )      )




    def   __lowerCamelCase				( self    ,					lowercase__      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	"lower newer"
          SCREAMING_SNAKE_CASE_				:      str					=	"lower newer"
          return input_text, output_text




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	self.tokenizer_class(vocab_file=self.vocab_file    ,					merges_file=self.merges_file      )

          SCREAMING_SNAKE_CASE_				:      int					=	"lower"
          SCREAMING_SNAKE_CASE_				:      List[str]					=	["low", "er</w>"]
          SCREAMING_SNAKE_CASE_				:      Optional[int]					=	tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokens + ["<unk>"]
          SCREAMING_SNAKE_CASE_				:      Dict					=	[16, 17, 23]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__      )    ,					lowercase__      )




    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          if not self.test_rust_tokenizer:
                return

          SCREAMING_SNAKE_CASE_				:      Dict					=	self.get_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	"lower,newer"

          SCREAMING_SNAKE_CASE_				:      int					=	tokenizer.tokenize(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      List[Any]					=	rust_tokenizer.tokenize(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      int					=	rust_tokenizer.encode(lowercase__    ,					add_special_tokens=lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )

          SCREAMING_SNAKE_CASE_				:      Any					=	self.get_rust_tokenizer()
          SCREAMING_SNAKE_CASE_				:      Optional[Any]					=	tokenizer.encode(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	rust_tokenizer.encode(lowercase__      )
          self.assertListEqual(lowercase__    ,					lowercase__      )







    @slow
    def   __lowerCamelCase				( self      ):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_				:      int					=	self.tokenizer_class.from_pretrained("allegro/herbert-base-cased"      )

          SCREAMING_SNAKE_CASE_				:      List[Any]					=	tokenizer.encode("konstruowanie sekwencji"    ,					add_special_tokens=lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Union[str, Any]					=	tokenizer.encode("konstruowanie wielu sekwencji"    ,					add_special_tokens=lowercase__      )

          SCREAMING_SNAKE_CASE_				:      List[str]					=	tokenizer.build_inputs_with_special_tokens(lowercase__      )
          SCREAMING_SNAKE_CASE_				:      Dict					=	tokenizer.build_inputs_with_special_tokens(lowercase__    ,					lowercase__      )

          assert encoded_sentence == [0] + text + [2]
          assert encoded_pair == [0] + text + [2] + text_a + [2]

