


'''simple docstring'''







import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a__  : int				=	logging.get_logger(__name__)

a__  : int				=	{
    'src_vocab_file': 'vocab-src.json',
    'tgt_vocab_file': 'vocab-tgt.json',
    'merges_file': 'merges.txt',
}

a__  : Any				=	{
    'src_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json'
    },
    'tgt_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json'
    },
    'merges_file': {'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt'},
}

a__  : Union[str, Any]				=	{'stas/tiny-wmt19-en-de': 1_0_2_4}
a__  : Union[str, Any]				=	{
    'stas/tiny-wmt19-en-de': {
        'langs': ['en', 'de'],
        'model_max_length': 1_0_2_4,
        'special_tokens_map_file': None,
        'full_tokenizer_file': None,
    }
}







def _UpperCamelCase	(					__A   )							->		Any:


      '''simple docstring'''



      UpperCamelCase__  =    set()
      UpperCamelCase__  =    word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)   )
            UpperCamelCase__  =    char
      return pairs







def _UpperCamelCase	(					__A   )							->		Union[str, Any]:


      '''simple docstring'''



      UpperCamelCase__  =    text.replace("，"				,      ","   )
      UpperCamelCase__  =    re.sub(R"。\s*"				,      ". "				,      __A   )
      UpperCamelCase__  =    text.replace("、"				,      ","   )
      UpperCamelCase__  =    text.replace("”"				,      "\""   )
      UpperCamelCase__  =    text.replace("“"				,      "\""   )
      UpperCamelCase__  =    text.replace("∶"				,      ":"   )
      UpperCamelCase__  =    text.replace("："				,      ":"   )
      UpperCamelCase__  =    text.replace("？"				,      "?"   )
      UpperCamelCase__  =    text.replace("《"				,      "\""   )
      UpperCamelCase__  =    text.replace("》"				,      "\""   )
      UpperCamelCase__  =    text.replace("）"				,      ")"   )
      UpperCamelCase__  =    text.replace("！"				,      "!"   )
      UpperCamelCase__  =    text.replace("（"				,      "("   )
      UpperCamelCase__  =    text.replace("；"				,      ";"   )
      UpperCamelCase__  =    text.replace("１"				,      "1"   )
      UpperCamelCase__  =    text.replace("」"				,      "\""   )
      UpperCamelCase__  =    text.replace("「"				,      "\""   )
      UpperCamelCase__  =    text.replace("０"				,      "0"   )
      UpperCamelCase__  =    text.replace("３"				,      "3"   )
      UpperCamelCase__  =    text.replace("２"				,      "2"   )
      UpperCamelCase__  =    text.replace("５"				,      "5"   )
      UpperCamelCase__  =    text.replace("６"				,      "6"   )
      UpperCamelCase__  =    text.replace("９"				,      "9"   )
      UpperCamelCase__  =    text.replace("７"				,      "7"   )
      UpperCamelCase__  =    text.replace("８"				,      "8"   )
      UpperCamelCase__  =    text.replace("４"				,      "4"   )
      UpperCamelCase__  =    re.sub(R"．\s*"				,      ". "				,      __A   )
      UpperCamelCase__  =    text.replace("～"				,      "~"   )
      UpperCamelCase__  =    text.replace("’"				,      "'"   )
      UpperCamelCase__  =    text.replace("…"				,      "..."   )
      UpperCamelCase__  =    text.replace("━"				,      "-"   )
      UpperCamelCase__  =    text.replace("〈"				,      "<"   )
      UpperCamelCase__  =    text.replace("〉"				,      ">"   )
      UpperCamelCase__  =    text.replace("【"				,      "["   )
      UpperCamelCase__  =    text.replace("】"				,      "]"   )
      UpperCamelCase__  =    text.replace("％"				,      "%"   )
      return text







def _UpperCamelCase	(					__A   )							->		Union[str, Any]:


      '''simple docstring'''



      UpperCamelCase__  =    []
      for char in text:
            UpperCamelCase__  =    unicodedata.category(__A   )
            if cat.startswith("C"   ):
                  continue
            output.append(__A   )
      return "".join(__A   )



class  lowercase_					(   a__       ):
    __UpperCAmelCase             =     VOCAB_FILES_NAMES
    __UpperCAmelCase             =     PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase             =     PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase             =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase             =     ['input_ids', 'attention_mask']


    def __init__(					self  , a=None  , a=None  , a=None  , a=None  , a=False  , a="<unk>"  , a="<s>"  , a="</s>"  , a="<pad>"  , **a  , ):
          super().__init__(
              langs=a  , src_vocab_file=a  , tgt_vocab_file=a  , merges_file=a  , do_lower_case=a  , unk_token=a  , bos_token=a  , sep_token=a  , pad_token=a  , **a  , )

          try:
                import sacremoses
          except ImportError:
                raise ImportError(
                    "You need to install sacremoses to use XLMTokenizer. "
                    "See https://pypi.org/project/sacremoses/ for installation."   )

          UpperCamelCase__  =    sacremoses

          UpperCamelCase__  =    src_vocab_file
          UpperCamelCase__  =    tgt_vocab_file
          UpperCamelCase__  =    merges_file
          UpperCamelCase__  =    do_lower_case

          # cache of sm.MosesPunctNormalizer instance
          UpperCamelCase__  =    {}
          # cache of sm.MosesTokenizer instance
          UpperCamelCase__  =    {}
          UpperCamelCase__  =    {}

          if langs and len(a   ) == 2:
                UpperCamelCase__ ,     UpperCamelCase__  =    langs
          else:
                raise ValueError(
                    f'''arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '''
                    "Usually that means that tokenizer can't find a mapping for the given model path "
                    "in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."   )

          with open(a  , encoding="utf-8"   ) as src_vocab_handle:
                UpperCamelCase__  =    json.load(a   )
          with open(a  , encoding="utf-8"   ) as tgt_vocab_handle:
                UpperCamelCase__  =    json.load(a   )
                UpperCamelCase__  =    {v: k for k, v in tgt_vocab.items()}
          with open(a  , encoding="utf-8"   ) as merges_handle:
                UpperCamelCase__  =    merges_handle.read().split("\n"   )[:-1]
          UpperCamelCase__  =    [tuple(merge.split()[:2]   ) for merge in merges]
          UpperCamelCase__  =    dict(zip(a  , range(len(a   )   )   )   )
          UpperCamelCase__  =    {}


    def        __a							(					self   ):
          return self.get_src_vocab()


    @property
    def        __a							(					self   ):
          return self.src_vocab_size


    def        __a							(					self  , a  , a   ):
          if lang not in self.cache_moses_punct_normalizer:
                UpperCamelCase__  =    self.sm.MosesPunctNormalizer(lang=a   )
                UpperCamelCase__  =    punct_normalizer
          return self.cache_moses_punct_normalizer[lang].normalize(a   )


    def        __a							(					self  , a  , a   ):
          if lang not in self.cache_moses_tokenizer:
                UpperCamelCase__  =    self.sm.MosesTokenizer(lang=a   )
                UpperCamelCase__  =    moses_tokenizer
          return self.cache_moses_tokenizer[lang].tokenize(
              a  , aggressive_dash_splits=a  , return_str=a  , escape=a   )


    def        __a							(					self  , a  , a   ):
          if lang not in self.cache_moses_detokenizer:
                UpperCamelCase__  =    self.sm.MosesDetokenizer(lang=a   )
                UpperCamelCase__  =    moses_detokenizer
          return self.cache_moses_detokenizer[lang].detokenize(a   )


    def        __a							(					self  , a  , a   ):
          UpperCamelCase__  =    replace_unicode_punct(a   )
          UpperCamelCase__  =    self.moses_punct_norm(a  , a   )
          UpperCamelCase__  =    remove_non_printing_char(a   )
          return text


    @property
    def        __a							(					self   ):
          return len(self.encoder   )


    @property
    def        __a							(					self   ):
          return len(self.decoder   )


    def        __a							(					self   ):
          return dict(self.encoder  , **self.added_tokens_encoder   )


    def        __a							(					self   ):
          return dict(self.decoder  , **self.added_tokens_decoder   )


    def        __a							(					self  , a   ):
          UpperCamelCase__  =    tuple(token[:-1]   ) + (token[-1] + "</w>",)
          if token in self.cache:
                return self.cache[token]
          UpperCamelCase__  =    get_pairs(a   )

          if not pairs:
                return token + "</w>"

          while True:
                UpperCamelCase__  =    min(a  , key=lambda a   : self.bpe_ranks.get(a  , float("inf"   )   )   )
                if bigram not in self.bpe_ranks:
                      break
                UpperCamelCase__ ,     UpperCamelCase__  =    bigram
                UpperCamelCase__  =    []
                UpperCamelCase__  =    0
                while i < len(a   ):
                      try:
                            UpperCamelCase__  =    word.index(a  , a   )
                      except ValueError:
                            new_word.extend(word[i:]   )
                            break
                      else:
                            new_word.extend(word[i:j]   )
                            UpperCamelCase__  =    j

                      if word[i] == first and i < len(a   ) - 1 and word[i + 1] == second:
                            new_word.append(first + second   )
                            i += 2
                      else:
                            new_word.append(word[i]   )
                            i += 1
                UpperCamelCase__  =    tuple(a   )
                UpperCamelCase__  =    new_word
                if len(a   ) == 1:
                      break
                else:
                      UpperCamelCase__  =    get_pairs(a   )
          UpperCamelCase__  =    " ".join(a   )
          if word == "\n  </w>":
                UpperCamelCase__  =    "\n</w>"
          UpperCamelCase__  =    word
          return word


    def        __a							(					self  , a  , a="en"  , a=False   ):
          UpperCamelCase__  =    self.src_lang

          if self.do_lower_case:
                UpperCamelCase__  =    text.lower()

          if bypass_tokenizer:
                UpperCamelCase__  =    text.split()
          else:
                UpperCamelCase__  =    self.moses_pipeline(a  , lang=a   )
                UpperCamelCase__  =    self.moses_tokenize(a  , lang=a   )

          UpperCamelCase__  =    []
          for token in text:
                if token:
                      split_tokens.extend(list(self.bpe(a   ).split(" "   )   )   )

          return split_tokens


    def        __a							(					self  , a   ):
          return self.encoder.get(a  , self.encoder.get(self.unk_token   )   )


    def        __a							(					self  , a   ):
          return self.decoder.get(a  , self.unk_token   )


    def        __a							(					self  , a   ):
          UpperCamelCase__  =    [t.replace(" "  , ""   ).replace("</w>"  , " "   ) for t in tokens]
          UpperCamelCase__  =    "".join(a   ).split()
          # detokenize
          UpperCamelCase__  =    self.moses_detokenize(a  , self.tgt_lang   )
          return text


    def        __a							(					self  , a  , a = None   ):
          UpperCamelCase__  =    [self.sep_token_id]

          # no bos used in fairseq
          if token_ids_a is None:
                return token_ids_a + sep
          return token_ids_a + sep + token_ids_a + sep


    def        __a							(					self  , a  , a = None  , a = False   ):

          if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=a  , token_ids_a=a  , already_has_special_tokens=a   )
          # no bos used in fairseq
          if token_ids_a is not None:
                return ([0] * len(a   )) + [1] + ([0] * len(a   )) + [1]
          return ([0] * len(a   )) + [1]


    def        __a							(					self  , a  , a = None   ):
          UpperCamelCase__  =    [self.sep_token_id]

          # no bos used in fairseq
          if token_ids_a is None:
                return len(token_ids_a + sep   ) * [0]
          return len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]


    def        __a							(					self  , a  , a = None   ):
          if not os.path.isdir(a   ):
                logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
                return

          UpperCamelCase__  =    os.path.join(
              a  , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["src_vocab_file"]   )
          UpperCamelCase__  =    os.path.join(
              a  , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["tgt_vocab_file"]   )
          UpperCamelCase__  =    os.path.join(
              a  , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )

          with open(a  , "w"  , encoding="utf-8"   ) as f:
                f.write(json.dumps(self.encoder  , indent=2  , sort_keys=a  , ensure_ascii=a   ) + "\n"   )

          with open(a  , "w"  , encoding="utf-8"   ) as f:
                UpperCamelCase__  =    {v: k for k, v in self.decoder.items()}
                f.write(json.dumps(a  , indent=2  , sort_keys=a  , ensure_ascii=a   ) + "\n"   )

          UpperCamelCase__  =    0
          with open(a  , "w"  , encoding="utf-8"   ) as writer:
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  , key=lambda a   : kv[1]   ):
                      if index != token_index:
                            logger.warning(
                                f'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
                                " Please check that the tokenizer is not corrupted!"   )
                            UpperCamelCase__  =    token_index
                      writer.write(" ".join(a   ) + "\n"   )
                      index += 1

          return src_vocab_file, tgt_vocab_file, merges_file


    def __getstate__(					self   ):
          UpperCamelCase__  =    self.__dict__.copy()
          UpperCamelCase__  =    None
          return state


    def __setstate__(					self  , a   ):
          UpperCamelCase__  =    d

          try:
                import sacremoses
          except ImportError:
                raise ImportError(
                    "You need to install sacremoses to use XLMTokenizer. "
                    "See https://pypi.org/project/sacremoses/ for installation."   )

          UpperCamelCase__  =    sacremoses









'''simple docstring'''







import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


if TYPE_CHECKING:
  import sqlitea

  import sqlalchemy


a__  : Optional[int]				=	datasets.utils.logging.get_logger(__name__)
@dataclass
class  lowercase_					(   datasets.BuilderConfig       ):
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     True
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     None
    __UpperCAmelCase             =     10000
    __UpperCAmelCase             =     None


    def        __a							(					self   ):
          if self.sql is None:
                raise ValueError("sql must be specified"   )
          if self.con is None:
                raise ValueError("con must be specified"   )


    def        __a							(					self  , a  , a = None  , ):
          UpperCamelCase__  =    config_kwargs.copy()
          # We need to stringify the Selectable object to make its hash deterministic

          # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html
          UpperCamelCase__  =    config_kwargs["sql"]
          if not isinstance(a  , a   ):
                if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules:
                      import sqlalchemy

                      if isinstance(a  , sqlalchemy.sql.Selectable   ):
                            UpperCamelCase__  =    sqlalchemy.create_engine(config_kwargs["con"].split("://"   )[0] + "://"   )
                            UpperCamelCase__  =    str(sql.compile(dialect=engine.dialect   )   )
                            UpperCamelCase__  =    sql_str
                      else:
                            raise TypeError(
                                f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(a   )}: {sql}'''   )
                else:
                      raise TypeError(
                          f'''Supported types for \'sql\' are string and sqlalchemy.sql.Selectable but got {type(a   )}: {sql}'''   )
          UpperCamelCase__  =    config_kwargs["con"]
          if not isinstance(a  , a   ):
                UpperCamelCase__  =    id(a   )
                logger.info(
                    f'''SQL connection \'con\' of type {type(a   )} couldn\'t be hashed properly. To enable hashing, specify \'con\' as URI string instead.'''   )

          return super().create_config_id(a  , custom_features=a   )


    @property
    def        __a							(					self   ):
          UpperCamelCase__  =    {
              "index_col": self.index_col,
              "columns": self.columns,
              "params": self.params,
              "coerce_float": self.coerce_float,
              "parse_dates": self.parse_dates,
          }
          return pd_read_sql_kwargs

class  lowercase_					(   datasets.ArrowBasedBuilder       ):
    __UpperCAmelCase             =     SqlConfig


    def        __a							(					self   ):
          return datasets.DatasetInfo(features=self.config.features   )


    def        __a							(					self  , a   ):
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN  , gen_kwargs={}   )]


    def        __a							(					self  , a   ):
          if self.config.features is not None:
                UpperCamelCase__  =    self.config.features.arrow_schema
                if all(not require_storage_cast(a   ) for feature in self.config.features.values()   ):
                      # cheaper cast
                      UpperCamelCase__  =    pa.Table.from_arrays([pa_table[field.name] for field in schema]  , schema=a   )
                else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      UpperCamelCase__  =    table_cast(a  , a   )
          return pa_table


    def        __a							(					self   ):
          UpperCamelCase__  =    self.config.chunksize
          UpperCamelCase__  =    pd.read_sql(
              self.config.sql  , self.config.con  , chunksize=a  , **self.config.pd_read_sql_kwargs   )
          UpperCamelCase__  =    [sql_reader] if chunksize is None else sql_reader
          for chunk_idx, df in enumerate(a   ):
                UpperCamelCase__  =    pa.Table.from_pandas(a   )
                yield chunk_idx, self._cast_table(a   )





