


'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


A__		: Dict												=  datasets.utils.logging.get_logger(__name__)





@dataclass
class        UpperCAmelCase_  (datasets.BuilderConfig  ):





    """simple docstring"""






    lowerCamelCase  : Optional[datasets.Features]												=      None
    lowerCamelCase  : str												=      "utf-8"
    lowerCamelCase  : Optional[str]												=      None
    lowerCamelCase  : int												=      1_0 << 2_0  # 10MB
    lowerCamelCase  : bool												=      False
    lowerCamelCase  : str												=      "line"







class        UpperCAmelCase_  (datasets.ArrowBasedBuilder  ):





    """simple docstring"""






    lowerCamelCase  : Optional[Any]												=      TextConfig







    def 				lowercase_    (							self     )    ->					int:
         return datasets.DatasetInfo(features=self.config.features     )







    def 				lowercase_    (							self      ,					SCREAMING_SNAKE_CASE_     )    ->					List[Any]:
         if not self.config.data_files:
              raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'     )
         __lowerCamelCase       :     Optional[Any]        =		dl_manager.download_and_extract(self.config.data_files     )
         if isinstance(SCREAMING_SNAKE_CASE_      ,					(str, list, tuple)     ):
              __lowerCamelCase       :     List[Any]        =		data_files
              if isinstance(SCREAMING_SNAKE_CASE_      ,					SCREAMING_SNAKE_CASE_     ):
                   __lowerCamelCase       :     Any        =		[files]
              __lowerCamelCase       :     Tuple        =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE_     ) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,					gen_kwargs={'files': files}     )]
         __lowerCamelCase       :     str        =		[]
         for split_name, files in data_files.items():
              if isinstance(SCREAMING_SNAKE_CASE_      ,					SCREAMING_SNAKE_CASE_     ):
                   __lowerCamelCase       :     Tuple        =		[files]
              __lowerCamelCase       :     Optional[Any]        =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE_     ) for file in files]
              splits.append(datasets.SplitGenerator(name=SCREAMING_SNAKE_CASE_      ,					gen_kwargs={'files': files}     )     )
         return splits







    def 				lowercase_    (							self      ,					SCREAMING_SNAKE_CASE_     )    ->					pa.Table:
         if self.config.features is not None:
              __lowerCamelCase       :     Union[str, Any]        =		self.config.features.arrow_schema
              if all(not require_storage_cast(SCREAMING_SNAKE_CASE_     ) for feature in self.config.features.values()     ):
                   # cheaper cast
                   __lowerCamelCase       :     Any        =		pa_table.cast(SCREAMING_SNAKE_CASE_     )
              else:
                   # more expensive cast; allows str <-> int/float or str to Audio for example
                   __lowerCamelCase       :     str        =		table_cast(SCREAMING_SNAKE_CASE_      ,					SCREAMING_SNAKE_CASE_     )
              return pa_table
         else:
              return pa_table.cast(pa.schema({'text': pa.string()}     )     )







    def 				lowercase_    (							self      ,					SCREAMING_SNAKE_CASE_     )    ->					Optional[Any]:
         __lowerCamelCase       :     str        =		list(self.config.features     ) if self.config.features is not None else ['text']
         for file_idx, file in enumerate(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE_     )     ):
              # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
              with open(SCREAMING_SNAKE_CASE_      ,					encoding=self.config.encoding      ,					errors=self.config.errors     ) as f:
                   if self.config.sample_by == "line":
                        __lowerCamelCase       :     List[Any]        =		0
                        while True:
                             __lowerCamelCase       :     Optional[int]        =		f.read(self.config.chunksize     )
                             if not batch:
                                  break
                             batch += f.readline()  # finish current line
                             # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                             __lowerCamelCase       :     List[Any]        =		StringIO(SCREAMING_SNAKE_CASE_     ).readlines()
                             if not self.config.keep_linebreaks:
                                  __lowerCamelCase       :     Tuple        =		[line.rstrip('\n'     ) for line in batch]
                             __lowerCamelCase       :     int        =		pa.Table.from_arrays([pa.array(SCREAMING_SNAKE_CASE_     )]      ,					names=SCREAMING_SNAKE_CASE_     )
                             # Uncomment for debugging (will print the Arrow table size and elements)
                             # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                             # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                             yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE_     )
                             batch_idx += 1
                   elif self.config.sample_by == "paragraph":
                        __lowerCamelCase       :     List[str]        =		0
                        __lowerCamelCase       :     str        =		''
                        while True:
                             __lowerCamelCase       :     Dict        =		f.read(self.config.chunksize     )
                             if not new_batch:
                                  break
                             batch += new_batch
                             batch += f.readline()  # finish current line
                             __lowerCamelCase       :     List[Any]        =		batch.split('\n\n'     )
                             __lowerCamelCase       :     List[str]        =		pa.Table.from_arrays(
                                 [pa.array([example for example in batch[:-1] if example]     )]      ,					names=SCREAMING_SNAKE_CASE_     )
                             # Uncomment for debugging (will print the Arrow table size and elements)
                             # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                             # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                             yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE_     )
                             batch_idx += 1
                             __lowerCamelCase       :     Dict        =		batch[-1]
                        if batch:
                             __lowerCamelCase       :     int        =		pa.Table.from_arrays([pa.array([batch]     )]      ,					names=SCREAMING_SNAKE_CASE_     )
                             yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE_     )
                   elif self.config.sample_by == "document":
                        __lowerCamelCase       :     List[Any]        =		f.read()
                        __lowerCamelCase       :     Optional[int]        =		pa.Table.from_arrays([pa.array([text]     )]      ,					names=SCREAMING_SNAKE_CASE_     )
                        yield file_idx, self._cast_table(SCREAMING_SNAKE_CASE_     )










'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


A__		: int												=  logging.get_logger(__name__)

A__		: Union[str, Any]												=  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

A__		: int												=  {
    """vocab_file""": {
        """microsoft/mpnet-base""": """https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """microsoft/mpnet-base""": """https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json""",
    },
}

A__		: int												=  {
    """microsoft/mpnet-base""": 512,
}

A__		: Optional[Any]												=  {
    """microsoft/mpnet-base""": {"""do_lower_case""": True},
}





class        UpperCAmelCase_  (_UpperCAmelCase  ):





    """simple docstring"""






    lowerCamelCase  : int												=      VOCAB_FILES_NAMES
    lowerCamelCase  : int												=      PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase  : List[Any]												=      PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase  : int												=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase  : Tuple												=      MPNetTokenizer
    lowerCamelCase  : List[Any]												=      ['input_ids', 'attention_mask']







    def __init__(							self      ,					SCREAMING_SNAKE_CASE_=None      ,					SCREAMING_SNAKE_CASE_=None      ,					SCREAMING_SNAKE_CASE_=True      ,					SCREAMING_SNAKE_CASE_="<s>"      ,					SCREAMING_SNAKE_CASE_="</s>"      ,					SCREAMING_SNAKE_CASE_="</s>"      ,					SCREAMING_SNAKE_CASE_="<s>"      ,					SCREAMING_SNAKE_CASE_="[UNK]"      ,					SCREAMING_SNAKE_CASE_="<pad>"      ,					SCREAMING_SNAKE_CASE_="<mask>"      ,					SCREAMING_SNAKE_CASE_=True      ,					SCREAMING_SNAKE_CASE_=None      ,					**SCREAMING_SNAKE_CASE_      ,					)    ->					Any:
         super().__init__(
             SCREAMING_SNAKE_CASE_      ,					tokenizer_file=SCREAMING_SNAKE_CASE_      ,					do_lower_case=SCREAMING_SNAKE_CASE_      ,					bos_token=SCREAMING_SNAKE_CASE_      ,					eos_token=SCREAMING_SNAKE_CASE_      ,					sep_token=SCREAMING_SNAKE_CASE_      ,					cls_token=SCREAMING_SNAKE_CASE_      ,					unk_token=SCREAMING_SNAKE_CASE_      ,					pad_token=SCREAMING_SNAKE_CASE_      ,					mask_token=SCREAMING_SNAKE_CASE_      ,					tokenize_chinese_chars=SCREAMING_SNAKE_CASE_      ,					strip_accents=SCREAMING_SNAKE_CASE_      ,					**SCREAMING_SNAKE_CASE_      ,					)

         __lowerCamelCase       :     Union[str, Any]        =		json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
         if (
             pre_tok_state.get('lowercase'      ,					SCREAMING_SNAKE_CASE_     ) != do_lower_case
             or pre_tok_state.get('strip_accents'      ,					SCREAMING_SNAKE_CASE_     ) != strip_accents
         ):
              __lowerCamelCase       :     List[Any]        =		getattr(SCREAMING_SNAKE_CASE_      ,					pre_tok_state.pop('type'     )     )
              __lowerCamelCase       :     List[str]        =		do_lower_case
              __lowerCamelCase       :     str        =		strip_accents
              __lowerCamelCase       :     Union[str, Any]        =		pre_tok_class(**SCREAMING_SNAKE_CASE_     )

         __lowerCamelCase       :     Tuple        =		do_lower_case







    @property
    def 				lowercase_    (							self     )    ->					str:
         if self._mask_token is None:
              if self.verbose:
                   logger.error('Using mask_token, but it is not set yet.'     )
              return None
         return str(self._mask_token     )







    @mask_token.setter
    def 				lowercase_    (							self      ,					SCREAMING_SNAKE_CASE_     )    ->					Union[str, Any]:
         __lowerCamelCase       :     Any        =		AddedToken(SCREAMING_SNAKE_CASE_      ,					lstrip=SCREAMING_SNAKE_CASE_      ,					rstrip=SCREAMING_SNAKE_CASE_     ) if isinstance(SCREAMING_SNAKE_CASE_      ,					SCREAMING_SNAKE_CASE_     ) else value
         __lowerCamelCase       :     int        =		value







    def 				lowercase_    (							self      ,					SCREAMING_SNAKE_CASE_      ,					SCREAMING_SNAKE_CASE_=None     )    ->					Union[str, Any]:
         __lowerCamelCase       :     Dict        =		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
              return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







    def 				lowercase_    (							self      ,					SCREAMING_SNAKE_CASE_      ,					SCREAMING_SNAKE_CASE_ = None     )    ->					List[int]:
         __lowerCamelCase       :     List[Any]        =		[self.sep_token_id]
         __lowerCamelCase       :     List[Any]        =		[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep     ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]







    def 				lowercase_    (							self      ,					SCREAMING_SNAKE_CASE_      ,					SCREAMING_SNAKE_CASE_ = None     )    ->					Tuple[str]:
         __lowerCamelCase       :     Optional[Any]        =		self._tokenizer.model.save(SCREAMING_SNAKE_CASE_      ,					name=SCREAMING_SNAKE_CASE_     )
         return tuple(SCREAMING_SNAKE_CASE_     )






