






import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
  import sqlitea

  import sqlalchemy


__a :    str								=				datasets.utils.logging.get_logger(__name__)


@dataclass
class  __lowercase (		datasets.BuilderConfig					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			None
     SCREAMING_SNAKE_CASE				  =			None
     SCREAMING_SNAKE_CASE				  =			None
     SCREAMING_SNAKE_CASE				  =			True
     SCREAMING_SNAKE_CASE				  =			None
     SCREAMING_SNAKE_CASE				  =			None
     SCREAMING_SNAKE_CASE				  =			None
     SCREAMING_SNAKE_CASE				  =			1_00_00
     SCREAMING_SNAKE_CASE				  =			None




     def 						lowerCAmelCase_       (			self    :     List[Any]			):


         """simple docstring"""



         if self.sql is None:
             raise ValueError("""sql must be specified"""			)
         if self.con is None:
             raise ValueError("""con must be specified"""			)




     def 						lowerCAmelCase_       (			self    :     Optional[int]	,		UpperCamelCase_    :     Tuple	,		UpperCamelCase_    :     List[Any] = None	,		):


         """simple docstring"""



         __A             =		config_kwargs.copy()
         # We need to stringify the Selectable object to make its hash deterministic

         # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
         __A             =		config_kwargs["""sql"""]
         if not isinstance(snake_case__	,		snake_case__			):
             if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                 import sqlalchemy

                 if isinstance(snake_case__	,		sqlalchemy.sql.Selectable			):
                     __A             =		sqlalchemy.create_engine(config_kwargs["""con"""].split("""://"""			)[0] + """://"""			)
                     __A             =		str(sql.compile(dialect=engine.dialect			)			)
                     __A             =		sql_str
                 else:
                     raise TypeError(
                         F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(snake_case__			)}: {sql}"			)
             else:
                 raise TypeError(
                     F"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(snake_case__			)}: {sql}"			)
         __A             =		config_kwargs["""con"""]
         if not isinstance(snake_case__	,		snake_case__			):
             __A             =		id(snake_case__			)
             logger.info(
                 F"SQL connection 'con' of type {type(snake_case__			)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead."			)

         return super().create_config_id(snake_case__	,		custom_features=snake_case__			)





     @property
     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         __A             =		{
             """index_col""": self.index_col,
             """columns""": self.columns,
             """params""": self.params,
             """coerce_float""": self.coerce_float,
             """parse_dates""": self.parse_dates,
         }
         return pd_read_sql_kwargs





class  __lowercase (		datasets.ArrowBasedBuilder					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			SqlConfig




     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         return datasets.DatasetInfo(features=self.config.features			)




     def 						lowerCAmelCase_       (			self    :     int	,		UpperCamelCase_    :     str			):


         """simple docstring"""



         return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,		gen_kwargs={}			)]




     def 						lowerCAmelCase_       (			self    :     List[Any]	,		UpperCamelCase_    :     Optional[int]			):


         """simple docstring"""



         if self.config.features is not None:
             __A             =		self.config.features.arrow_schema
             if all(not require_storage_cast(snake_case__			) for feature in self.config.features.values()			):
                 # cheaper cast
                 __A             =		pa.Table.from_arrays([pa_table[field.name] for field in schema]	,		schema=snake_case__			)
             else:
                 # more expensive cast; allows str <-> int/float or str to Audio for example
                 __A             =		table_cast(snake_case__	,		snake_case__			)
         return pa_table





     def 						lowerCAmelCase_       (			self    :     str			):


         """simple docstring"""



         __A             =		self.config.chunksize
         __A             =		pd.read_sql(
             self.config.sql	,		self.config.con	,		chunksize=snake_case__	,		**self.config.pd_read_sql_kwargs			)
         __A             =		[sql_reader] if chunksize is None else sql_reader
         for chunk_idx, df in enumerate(snake_case__			):
             __A             =		pa.Table.from_pandas(snake_case__			)
             yield chunk_idx, self._cast_table(snake_case__			)












import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__a :    Optional[Any]								=				get_tests_dir("fixtures/spiece.model")


@require_sentencepiece
@require_tokenizers
class  __lowercase (		lowercase_     , unittest.TestCase					):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE				  =			ErnieMTokenizer
     SCREAMING_SNAKE_CASE				  =			False
     SCREAMING_SNAKE_CASE				  =			True
     SCREAMING_SNAKE_CASE				  =			False
     SCREAMING_SNAKE_CASE				  =			False




     def 						lowerCAmelCase_       (			self    :     Optional[Any]			):


         """simple docstring"""



         super().setUp()

         # We have a SentencePiece fixture for testing
         __A             =		ErnieMTokenizer(UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)
         tokenizer.save_pretrained(self.tmpdirname			)




     def 						lowerCAmelCase_       (			self    :     Optional[Any]	,		UpperCamelCase_    :     Tuple			):


         """simple docstring"""



         __A             =		"""this is a test"""
         __A             =		"""this is a test"""
         return input_text, output_text




     def 						lowerCAmelCase_       (			self    :     Optional[int]			):


         """simple docstring"""



         __A             =		"""<pad>"""
         __A             =		0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_			)	,		UpperCamelCase_			)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_			)	,		UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Tuple			):


         """simple docstring"""



         __A             =		list(self.get_tokenizer().get_vocab().keys()			)

         self.assertEqual(vocab_keys[0]	,		"""<pad>"""			)
         self.assertEqual(vocab_keys[1]	,		"""<unk>"""			)
         self.assertEqual(vocab_keys[-1]	,		"""▁eloquent"""			)
         self.assertEqual(len(UpperCamelCase_			)	,		30_000			)




     def 						lowerCAmelCase_       (			self    :     Tuple			):


         """simple docstring"""



         self.assertEqual(self.get_tokenizer().vocab_size	,		30_000			)




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         if not self.test_rust_tokenizer:
             return

         __A             =		self.get_tokenizer()
         __A             =		self.get_rust_tokenizer()

         __A             =		"""I was born in 92000, and this is falsé."""

         __A             =		tokenizer.tokenize(UpperCamelCase_			)
         __A             =		rust_tokenizer.tokenize(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)

         __A             =		tokenizer.encode(UpperCamelCase_	,		add_special_tokens=UpperCamelCase_			)
         __A             =		rust_tokenizer.encode(UpperCamelCase_	,		add_special_tokens=UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)

         __A             =		self.get_rust_tokenizer()
         __A             =		tokenizer.encode(UpperCamelCase_			)
         __A             =		rust_tokenizer.encode(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		UpperCamelCase_			)




     def 						lowerCAmelCase_       (			self    :     Union[str, Any]			):


         """simple docstring"""



         __A             =		ErnieMTokenizer(UpperCamelCase_	,		do_lower_case=UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)

         __A             =		tokenizer.tokenize("""This is a test"""			)
         self.assertListEqual(UpperCamelCase_	,		["""▁this""", """▁is""", """▁a""", """▁test"""]			)

         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_			)	,		[48, 25, 21, 1_289]			)

         __A             =		tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
         # ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
         self.assertListEqual(
             UpperCamelCase_	,		["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]			)
         __A             =		tokenizer.convert_tokens_to_ids(UpperCamelCase_			)
         self.assertListEqual(UpperCamelCase_	,		[31, 23, 386, 19, 518, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]			)

         __A             =		tokenizer.convert_ids_to_tokens(UpperCamelCase_			)
         self.assertListEqual(
             UpperCamelCase_	,		["""▁i""", """▁was""", """▁born""", """▁in""", """9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""]	,		)




     def 						lowerCAmelCase_       (			self    :     List[str]			):


         """simple docstring"""



         __A             =		ErnieMTokenizer(UpperCamelCase_	,		unk_token="""<unk>"""	,		pad_token="""<pad>"""			)

         __A             =		tokenizer.encode("""sequence builders"""			)
         __A             =		tokenizer.encode("""multi-sequence build"""			)

         __A             =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase_			)
         __A             =		tokenizer.build_inputs_with_special_tokens(UpperCamelCase_	,		UpperCamelCase_			)

         assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
         assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
             tokenizer.sep_token_id
         ] + text_a + [tokenizer.sep_token_id]





     @slow
     def 						lowerCAmelCase_       (			self    :     List[str]			):


         """simple docstring"""



         __A             =		{"""input_ids""": [[0, 11_062, 82_772, 7, 15, 82_772, 538, 51_529, 237, 17_198, 1_290, 206, 9, 215_175, 1_314, 136, 17_198, 1_290, 206, 9, 56_359, 42, 122_009, 9, 16_466, 16, 87_344, 4_537, 9, 4_717, 78_381, 6, 159_958, 7, 15, 24_480, 618, 4, 527, 22_693, 9, 304, 4, 2_777, 24_480, 9_874, 4, 43_523, 594, 4, 803, 18_392, 33_189, 18, 4, 43_523, 24_447, 5, 5, 5, 16, 100, 24_955, 83_658, 9_626, 144_057, 15, 839, 22_335, 16, 136, 24_955, 83_658, 83_479, 15, 39_102, 724, 16, 678, 645, 6_460, 1_328, 4_589, 42, 122_009, 115_774, 23, 3_559, 1_328, 46_876, 7, 136, 53_894, 1_940, 42_227, 41_159, 17_721, 823, 425, 4, 27_512, 98_722, 206, 136, 5_531, 4_970, 919, 17_336, 5, 2], [0, 20_080, 618, 83, 82_775, 47, 479, 9, 1_517, 73, 53_894, 333, 80_581, 110_117, 18_811, 5_256, 1_295, 51, 152_526, 297, 7_986, 390, 124_416, 538, 35_431, 214, 98, 15_044, 25_737, 136, 7_108, 43_701, 23, 756, 135_355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63_773, 119_455, 6, 147_797, 88_203, 7, 645, 70, 21, 3_285, 10_269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=UpperCamelCase_	,		model_name="""susnato/ernie-m-base_pytorch"""	,		sequences=[
                 """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
                 """general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
                 """Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained """
                 """models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
                 """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
                 """conditioning on both left and right context in all layers.""",
                 """The quick brown fox jumps over the lazy dog.""",
             ]	,		)




